20 changes: 20 additions & 0 deletions .github/workflows/ibis-backends-skip-helper.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# This job intentionally collides with the Backend job in `ibis-backends.yml`
# that would be skipped because the paths are ignored. This is so the
# `Backends` job isn't stuck in "expected" forever when it should be skipped
name: Backends
on:
push:
paths:
- "docs/**"
branches:
- master
pull_request:
paths:
- "docs/**"
branches:
- master
jobs:
backends:
runs-on: ubuntu-latest
steps:
- run: echo "No build required"
489 changes: 164 additions & 325 deletions .github/workflows/ibis-backends.yml
View file

Large diffs are not rendered by default.

162 changes: 102 additions & 60 deletions .github/workflows/ibis-docs-lint.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# vim: filetype=yaml
name: Ibis Docs and Linting
name: Docs/Linting/Benchmarks

on:
push:
Expand All @@ -19,26 +19,26 @@ jobs:
if: ${{ github.event_name == 'pull_request' }}
steps:
- name: checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: install nix
uses: cachix/install-nix-action@v16
uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: lint commits
run: nix shell -L --keep-going -f '<nixpkgs>' commitlint -c commitlint --from=${{ github.event.pull_request.base.sha }} --to=${{ github.sha }} --verbose
run: nix shell -f '<nixpkgs>' commitlint -c commitlint --from=${{ github.event.pull_request.base.sha }} --to=${{ github.sha }} --verbose

lint:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: install nix
uses: cachix/install-nix-action@v16
uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small

Expand All @@ -50,19 +50,70 @@ jobs:
extraPullNames: nix-community,poetry2nix

- name: pre-commit checks
run: nix-shell --pure --keep-going --run 'pre-commit run --all-files'
run: nix develop -f shell.nix --ignore-environment --keep-going -c pre-commit run -a

docs:
name: Docs
benchmarks:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: install python
uses: actions/setup-python@v3
id: install_python
with:
python-version: "3.10"

- name: install system dependencies
run: sudo apt-get install -qq -y build-essential krb5-config libkrb5-dev

- uses: syphar/restore-virtualenv@v1
with:
path: ibis
requirement_files: poetry.lock
custom_cache_key_element: benchmarks-${{ steps.install_python.outputs.python-version }}

- uses: syphar/restore-pip-download-cache@v1
with:
requirement_files: poetry.lock
custom_cache_key_element: benchmarks-${{ steps.install_python.outputs.python-version }}

- run: python -m pip install --upgrade pip poetry

- name: install ibis
run: poetry install --extras all

- run: mkdir .benchmarks

- name: benchmark
run: poetry run pytest --benchmark-only --benchmark-json .benchmarks/output.json ibis/tests/benchmarks

- uses: tibdex/github-app-token@v1
if: ${{ github.event_name != 'pull_request' }}
id: generate-token
with:
app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}

- uses: benchmark-action/github-action-benchmark@v1
if: ${{ github.event_name != 'pull_request' }}
with:
tool: pytest
github-token: ${{ steps.generate-token.outputs.token }}
output-file-path: .benchmarks/output.json
benchmark-data-dir-path: bench
auto-push: true
comment-on-alert: true
alert-threshold: "300%"

docs:
runs-on: ubuntu-latest
needs:
# wait on benchmarks to prevent a race condition when pushing to the
# gh-pages branch
- benchmarks
steps:
- name: install nix
uses: cachix/install-nix-action@v16
uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small

Expand All @@ -73,92 +124,83 @@ jobs:
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix

- name: build docs
working-directory: ibis
run: nix-shell --pure --run 'make -C docs html BUILDDIR=web/docs'

- name: build website
working-directory: ibis
run: nix-shell --pure --run 'mkdocs build -f docs/mkdocs.yml'

- name: Add config to docs
working-directory: ibis
run: |
set -euo pipefail
touch docs/site/.nojekyll
echo "ibis-project.org" > docs/site/CNAME
- name: Generate a GitHub token
if: ${{ github.event_name == 'push' }}
uses: tibdex/github-app-token@v1
id: generate_token
with:
app_id: ${{ secrets.DOCS_BOT_APP_ID }}
private_key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }}
repository: ibis-project/ibis-project.org

- name: checkout
uses: actions/checkout@v2
if: ${{ github.event_name == 'push' }}
uses: actions/checkout@v3
with:
repository: ibis-project/ibis-project.org
fetch-depth: 0
token: ${{ steps.generate_token.outputs.token }}
path: ibis-project.org

- name: checkout
uses: actions/checkout@v2
if: ${{ github.event_name != 'push' }}
with:
repository: ibis-project/ibis-project.org
path: ibis-project.org

- name: Copy docbuild to checkout
working-directory: ibis
run: |
set -euo pipefail
# the trailing slash matters here; it means "everything underneath
# docbuild, but not docbuild itself"
rsync --delete --exclude=.git -avz docs/site/ ../ibis-project.org
uses: actions/checkout@v3

- name: Configure git info
working-directory: ibis-project.org
if: ${{ github.event_name == 'push' }}
run: |
set -euo pipefail
git config user.name 'ibis-docs-bot[bot]'
git config user.email 'ibis-docs-bot[bot]@users.noreply.github.com'
- name: Commit docs
working-directory: ibis-project.org
- name: build docs
if: ${{ github.event_name != 'push' }}
run: nix develop -f shell.nix --ignore-environment --keep-going -c mkdocs build

- name: verify internal links
if: ${{ github.event_name != 'push' }}
run: nix develop -f shell.nix --ignore-environment --keep-going -c just checklinks --offline --no-progress

- name: Pull gh-pages changes
if: ${{ github.event_name == 'push' }}
run: |
set -euo pipefail
git add .
git commit -am 'docs: ibis-project/ibis@${{ github.sha }}' || true
git fetch origin gh-pages
git update-ref refs/heads/gh-pages "$(git rev-parse origin/gh-pages)"
- name: tag docs if this is a release
if: ${{ startsWith(github.ref, 'refs/tags') }}
working-directory: ibis-project.org
- name: build and push dev docs
if: ${{ github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/') }}
run: |
set -euo pipefail
git tag "${GITHUB_REF##*/}"
nix develop -f shell.nix --keep-going -c \
mic deploy \
--push \
--rebase \
--prefix docs \
--message 'docs(dev): ibis@${{ github.sha }}' \
dev
- name: Push docs
if: ${{ github.event_name == 'push' }}
working-directory: ibis-project.org
run: git push --tags -f origin master
- name: build and push docs on tag
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
run: |
set -euo pipefail
nix develop -f shell.nix --keep-going -c \
mic deploy \
--push \
--rebase \
--prefix docs \
--message "docs(release): ibis@${GITHUB_REF_NAME}" \
"${GITHUB_REF_NAME}" latest
simulate_release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
fetch-depth: 0

- uses: cachix/install-nix-action@v16
- uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small

Expand Down
20 changes: 20 additions & 0 deletions .github/workflows/ibis-main-skip-helper.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# This job intentionally collides with the Ibis job in `ibis-main.yml`
# that would be skipped because the paths are ignored. This is so the `Ibis`
# job isn't stuck in "expected" forever when it should be skipped
name: Ibis
on:
push:
paths:
- "docs/**"
branches:
- master
pull_request:
paths:
- "docs/**"
branches:
- master
jobs:
nix-lint:
runs-on: ubuntu-latest
steps:
- run: echo "No build required"
105 changes: 36 additions & 69 deletions .github/workflows/ibis-main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ on:
push:
# Skip the test suite if all changes are in the docs directory
paths-ignore:
- 'docs/**'
- "docs/**"
branches:
- master
pull_request:
# Skip the test suite if all changes are in the docs directory
paths-ignore:
- 'docs/**'
- "docs/**"
branches:
- master

Expand All @@ -20,36 +20,28 @@ concurrency:
cancel-in-progress: true

jobs:
nix:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
python-version:
- "3.7"
- "3.8"
- "3.9"
- "3.10"
nix-lint:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: install nix
uses: cachix/install-nix-action@v16
uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: setup cachix
uses: cachix/cachix-action@v10
with:
name: ibis
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix

- name: nix build and run tests
run: nix build --keep-going --print-build-logs --file . --argstr python ${{ matrix.python-version }}
- name: nixpkgs-fmt
run: nix shell -f ./nix nixpkgs-fmt -c nixpkgs-fmt --check $(find . -name '*.nix' -and \( -not -wholename '*nix/sources.nix' \))

- name: nix-linter
run: nix shell -f ./nix nix-linter -c nix-linter $(find . -name '*.nix' -and \( -not -wholename '*nix/sources.nix' \))

test_no_backends:
name: Test ${{ matrix.os }} python-${{ matrix.python-version }}
Expand All @@ -61,16 +53,15 @@ jobs:
- ubuntu-latest
- windows-latest
python-version:
- "3.7"
- "3.8"
- "3.9"
- "3.10"
steps:
- name: checkout
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: install python
uses: actions/setup-python@v2
uses: actions/setup-python@v3
id: install_python
with:
python-version: ${{ matrix.python-version }}
Expand All @@ -85,61 +76,37 @@ jobs:
requirement_files: poetry.lock
custom_cache_key_element: no-backends-${{ steps.install_python.outputs.python-version }}

- name: install ${{ matrix.os }} system dependencies
if: ${{ matrix.os == 'ubuntu-latest' }}
run: |
set -euo pipefail
sudo apt-get update -y -q
sudo apt-get install -y -q build-essential graphviz
- name: install ${{ matrix.os }} system dependencies
if: ${{ matrix.os == 'windows-latest' }}
run: choco install graphviz

- run: python -m pip install --upgrade pip poetry

- name: install ibis
run: poetry install
run: poetry install --extras visualization

- uses: extractions/setup-just@v1

- name: run tests
shell: bash
run: ./ci/run_tests.sh ibis/tests
run: just ci-check -m core

- name: upload code coverage
if: success()
uses: codecov/codecov-action@v3
with:
flags: core,${{ runner.os }},python-${{ steps.install_python.outputs.python-version }}

- name: publish test report
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
if: success() || failure()
with:
name: no-backends-${{ matrix.os }}-${{ matrix.python-version }}
path: junit.xml

benchmarks:
name: Benchmark ${{ matrix.os }} python-${{ matrix.python-version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.10"
steps:
- name: checkout
uses: actions/checkout@v2

- name: install python
uses: actions/setup-python@v2
id: install_python
with:
python-version: ${{ matrix.python-version }}

- name: install system dependencies
run: sudo apt-get install -qq -y build-essential cmake krb5-config python-dev libkrb5-dev libboost-all-dev

- uses: syphar/restore-virtualenv@v1
with:
requirement_files: poetry.lock
custom_cache_key_element: benchmarks-${{ steps.install_python.outputs.python-version }}

- uses: syphar/restore-pip-download-cache@v1
with:
requirement_files: poetry.lock
custom_cache_key_element: benchmarks-${{ steps.install_python.outputs.python-version }}

- run: python -m pip install --upgrade pip poetry

- name: install ibis
run: poetry install --extras impala

- name: benchmark
run: |
set -euo pipefail
poetry run asv machine --yes
poetry run asv dev
35 changes: 35 additions & 0 deletions .github/workflows/nix-skip-helper.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# This job intentionally collides with the Nix job in `nix.yml`
# that would be skipped because the paths are ignored. This is so the `Nix`
# job isn't stuck in "expected" forever when it should be skipped
name: Nix

on:
push:
paths-ignore:
- "**/*.nix"
- "pyproject.toml"
- "poetry.lock"
- "nix/**"
branches:
- master
pull_request:
paths-ignore:
- "**/*.nix"
- "pyproject.toml"
- "poetry.lock"
- "nix/**"
branches:
- master

jobs:
nix:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
steps:
- run: echo "No build required"
54 changes: 54 additions & 0 deletions .github/workflows/nix.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# vim: filetype=yaml
name: Nix

on:
push:
paths:
- "**/*.nix"
- "pyproject.toml"
- "poetry.lock"
- "nix/**"
branches:
- master
pull_request:
paths:
- "**/*.nix"
- "pyproject.toml"
- "poetry.lock"
- "nix/**"
branches:
- master

concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true

jobs:
nix:
# TODO: we can't use macos-latest here until watchdog is fixed
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
steps:
- name: checkout
uses: actions/checkout@v3

- name: install nix
uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: setup cachix
uses: cachix/cachix-action@v10
with:
name: ibis
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix

- name: nix build and run tests
run: nix build --keep-going --print-build-logs --file . --argstr python ${{ matrix.python-version }}
4 changes: 2 additions & 2 deletions .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ jobs:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}

- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ steps.generate_token.outputs.token }}

- uses: cachix/install-nix-action@v16
- uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small
extra_nix_config: |
Expand Down
7 changes: 4 additions & 3 deletions .github/workflows/test-report.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: Test Report
on:
workflow_run:
workflows: ['Ibis', 'Backends']
workflows: ["Ibis", "Backends"]
types:
- completed
branches-ignore:
Expand All @@ -18,11 +18,12 @@ jobs:
uses: dawidd6/action-download-artifact@v2
with:
workflow: ${{ github.event.workflow_run.workflow_id }}
pr: ${{ github.event.pull_request.number }}
workflow_conclusion: completed
commit: ${{ github.event.workflow_run.head_sha }}
path: artifacts

- name: publish test report
uses: EnricoMi/publish-unit-test-result-action@v1
with:
commit: ${{ github.event.pull_request.head_sha }}
commit: ${{ github.event.workflow_run.head_sha }}
files: artifacts/**/junit.xml
12 changes: 6 additions & 6 deletions .github/workflows/update-deps.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3

- name: output dependency list
id: set-matrix
Expand All @@ -28,9 +28,9 @@ jobs:
strategy:
matrix: ${{ fromJSON(needs.generate_updates.outputs.matrix) }}
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3

- uses: cachix/install-nix-action@v16
- uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small

Expand Down Expand Up @@ -84,7 +84,7 @@ jobs:
basehead: ${{ steps.get_current_commit.outputs.rev }}...${{ steps.get_new_commit.outputs.rev }}
include-merge-commits: false

- uses: peter-evans/create-pull-request@v3
- uses: peter-evans/create-pull-request@v4
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
id: create_pr
with:
Expand All @@ -97,13 +97,13 @@ jobs:
body: ${{ steps.compare_commits.outputs.differences }}
labels: dependencies,autorebase:opt-in

- uses: juliangruber/approve-pull-request-action@v1.1.0
- uses: juliangruber/approve-pull-request-action@v1.1.1
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
with:
github-token: ${{ steps.generate_pr_approval_token.outputs.token }}
number: ${{ steps.create_pr.outputs.pull-request-number }}

- uses: peter-evans/enable-pull-request-automerge@v1
- uses: peter-evans/enable-pull-request-automerge@v2
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
with:
token: ${{ steps.generate_pr_token.outputs.token }}
Expand Down
27 changes: 17 additions & 10 deletions .github/workflows/update-setup-py.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,13 @@ jobs:
private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}

- name: checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
token: ${{ steps.generate_token.outputs.token }}
ref: ${{ github.event.pull_request.head.ref }}

- name: install nix
uses: cachix/install-nix-action@v16
uses: cachix/install-nix-action@v17
with:
nix_path: nixpkgs=channel:nixos-unstable-small

Expand All @@ -53,11 +54,17 @@ jobs:
name: ibis-squawk-bot[bot]
email: ibis-squawk-bot[bot]@users.noreply.github.com

- name: commit setup.py and push to pull request
uses: EndBug/add-and-commit@v7
with:
add: setup.py
author_name: ibis-squawk-bot[bot]
author_email: ibis-squawk-bot[bot]@users.noreply.github.com
message: "chore(setup.py): regenerate"
push: true
- name: commit setup.py and push to PR
run: |
set -euo pipefail
git add setup.py
if git commit -m 'chore(setup.py): regenerate'; then
# pull in case another commit happened in the meantime
#
# `ours` is actually the *other* changeset, not the current branch, per
# https://stackoverflow.com/a/3443225/564538
git pull --rebase -s recursive -X ours
git push
fi
19 changes: 16 additions & 3 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ Icon?
# documentation files
docs/source/generated
docs/source/generated-notebooks
*.org

# Ibis testing data
ci/ibis-testing-data*
Expand All @@ -74,7 +75,6 @@ docbuild

# nix generated files
.direnv
.pre-commit-config.yaml
result
result-*

Expand All @@ -86,5 +86,18 @@ result-*
requirements.txt

# generated mkdocs website
docs/site
docs/web/docs
.benchmarks
site

# tutorial data
geography.db
*.pbf
*.pbf.zst
ci/udf/.ninja_deps
ci/udf/.ninja_log
ci/udf/build.ninja
junit.xml
spark-warehouse
docs/backends/support_matrix.csv
__pycache__
tags
2 changes: 1 addition & 1 deletion .pep8speaks.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# File : .pep8speaks.yml

scanner:
diff_only: True # If True, errors caused by only the patch are shown
diff_only: True # If True, errors caused by only the patch are shown
linter: flake8

flake8:
Expand Down
127 changes: 127 additions & 0 deletions .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
default_stages:
- commit
repos:
- hooks:
- entry: black --check
exclude: ^$
files: ""
id: black
language: system
name: black
pass_filenames: true
stages:
- commit
types:
- python
- file
- python
types_or: []
- entry: flake8
exclude: ^$
files: ""
id: flake8
language: python
name: flake8
pass_filenames: true
stages:
- commit
types:
- python
types_or: []
- entry: absolufy-imports
name: absolufy
id: absolufy
language: python
files: ""
stages:
- commit
types:
- python
- entry: isort --check
exclude: ^$
files: ""
id: isort
language: python
name: isort
pass_filenames: true
stages:
- commit
types:
- file
- python
types_or:
- cython
- pyi
- python
- entry: pyupgrade --py38-plus
exclude: (setup.py)
files: ""
id: pyupgrade
language: system
name: pyupgrade
pass_filenames: true
stages:
- commit
types:
- python
types_or: []
- entry: shellcheck
exclude: ^$
files: \.sh$
id: shellcheck
language: system
name: shellcheck
pass_filenames: true
stages:
- commit
types:
- shell
types_or:
- file
- sh
- ash
- bash
- bats
- dash
- ksh
- entry: shfmt -i 2 -sr -d -s -l
exclude: ^$
files: \.sh$
id: shfmt
language: system
name: shfmt
pass_filenames: true
stages:
- commit
types:
- file
types_or:
- file
- entry: prettier --check
exclude: ^$
files: ""
id: prettier
language: system
name: prettier
pass_filenames: true
stages:
- commit
types:
- text
types_or:
- json
- toml
- yaml
- markdown
- entry: just --fmt --unstable --check
exclude: ^$
files: ^justfile$
id: just
language: system
name: just
pass_filenames: false
stages:
- commit
types:
- file
repo: local
12 changes: 12 additions & 0 deletions .prettierignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
.benchmarks
.direnv
.mypy_cache
.pytest_cache
nix/sources.json
result
result-*
docs/release_notes.md
docs/overrides/*.html
docs/SUMMARY.md
site
ci/udf/CMakeFiles
5 changes: 5 additions & 0 deletions .prettierrc.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
tabWidth = 2
semi = true
singleQuote = false
arrowParens = "avoid"
useTabs = false
32 changes: 30 additions & 2 deletions .releaserc.json
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
{
"branches": ["master"],
"tagFormat": "${version}",
"preset": "conventionalcommits",
"plugins": [
"@semantic-release/commit-analyzer",
"@semantic-release/release-notes-generator",
[
"@semantic-release/changelog",
{
"changelogTitle": "Release Notes\n---",
"changelogFile": "docs/web/release_notes.md"
"changelogFile": "docs/release_notes.md"
}
],
[
Expand All @@ -19,16 +20,43 @@
"publishCmd": "ci/release/publish.sh"
}
],
[
"@google/semantic-release-replace-plugin",
{
"replacements": [
{
"files": ["ibis/__init__.py"],
"from": "__version__ = \".*\"",
"to": "__version__ = \"${nextRelease.version}\"",
"results": [
{
"file": "ibis/__init__.py",
"hasChanged": true,
"numMatches": 1,
"numReplacements": 1
}
],
"countMatches": true
}
]
}
],
[
"@semantic-release/github",
{
"successComment": false,
"assets": ["dist/*.whl"]
}
],
[
"@semantic-release/git",
{
"assets": ["pyproject.toml", "docs/web/release_notes.md", "setup.py"],
"assets": [
"pyproject.toml",
"docs/release_notes.md",
"setup.py",
"ibis/__init__.py"
],
"message": "chore(release): ${nextRelease.version}"
}
]
Expand Down
3 changes: 3 additions & 0 deletions .rgignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
!.github
docs/source/tutorial/data/geography.json
docs/source/tutorial/data/geography.db
*.ipynb
56 changes: 26 additions & 30 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,17 @@
# Ibis: Python data analysis framework for Hadoop and SQL engines
# Ibis: Expressive analytics in Python at any scale

|Service|Status|
| -------------: | :---- |
| Documentation | [![Documentation Status](https://img.shields.io/badge/docs-docs.ibis--project.org-blue.svg)](http://ibis-project.org) |
| Conda packages | [![Anaconda-Server Badge](https://anaconda.org/conda-forge/ibis-framework/badges/version.svg)](https://anaconda.org/conda-forge/ibis-framework) |
| PyPI | [![PyPI](https://img.shields.io/pypi/v/ibis-framework.svg)](https://pypi.org/project/ibis-framework) |
| Ibis CI | [![Build status](https://github.com/ibis-project/ibis/actions/workflows/ibis-main.yml/badge.svg)](https://github.com/ibis-project/ibis/actions/workflows/ibis-main.yml?query=branch%3Amaster) |
| Backend CI | [![Build status](https://github.com/ibis-project/ibis/actions/workflows/ibis-backends.yml/badge.svg)](https://github.com/ibis-project/ibis/actions/workflows/ibis-backends.yml?query=branch%3Amaster) |
| Coverage | [![Codecov branch](https://img.shields.io/codecov/c/github/ibis-project/ibis/master.svg)](https://codecov.io/gh/ibis-project/ibis) |
| Service | Status |
| -------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Documentation | [![Documentation Status](https://img.shields.io/badge/docs-docs.ibis--project.org-blue.svg)](http://ibis-project.org) |
| Conda packages | [![Anaconda-Server Badge](https://anaconda.org/conda-forge/ibis-framework/badges/version.svg)](https://anaconda.org/conda-forge/ibis-framework) |
| PyPI | [![PyPI](https://img.shields.io/pypi/v/ibis-framework.svg)](https://pypi.org/project/ibis-framework) |
| Ibis CI | [![Build status](https://github.com/ibis-project/ibis/actions/workflows/ibis-main.yml/badge.svg)](https://github.com/ibis-project/ibis/actions/workflows/ibis-main.yml?query=branch%3Amaster) |
| Backend CI | [![Build status](https://github.com/ibis-project/ibis/actions/workflows/ibis-backends.yml/badge.svg)](https://github.com/ibis-project/ibis/actions/workflows/ibis-backends.yml?query=branch%3Amaster) |
| Coverage | [![Codecov branch](https://img.shields.io/codecov/c/github/ibis-project/ibis/master.svg)](https://codecov.io/gh/ibis-project/ibis) |


Ibis is a toolbox to bridge the gap between local Python environments, remote
storage, execution systems like Hadoop components (HDFS, Impala, Hive, Spark)
and SQL databases. Its goal is to simplify analytical workflows and make you
more productive.
Ibis is a Python library to help you write expressive analytics at any scale,
small to large. Its goal is to simplify analytical workflows and make you more
productive.

Install Ibis from PyPI with:

Expand All @@ -27,19 +25,17 @@ or from conda-forge with
conda install ibis-framework -c conda-forge
```

Ibis currently provides tools for interacting with the following systems:

- [Apache Impala](https://impala.apache.org/)
- [Apache Kudu](https://kudu.apache.org/)
- [Hadoop Distributed File System (HDFS)](https://hadoop.apache.org/)
- [PostgreSQL](https://www.postgresql.org/)
- [MySQL](https://www.mysql.com/)
- [SQLite](https://www.sqlite.org/)
- [Pandas](https://pandas.pydata.org/) [DataFrames](http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe)
- [Clickhouse](https://clickhouse.yandex)
- [BigQuery](https://cloud.google.com/bigquery)
- [OmniSciDB](https://www.omnisci.com)
- [PySpark](https://spark.apache.org)
- [Dask](https://dask.org/) (Experimental)

Learn more about using the library at http://ibis-project.org.
Ibis provides tools for interacting with the following systems:

- [Apache Impala](https://ibis-project.org/docs/latest/backends/Impala/)
- [Google BigQuery](https://github.com/ibis-project/ibis-bigquery)
- [ClickHouse](https://ibis-project.org/docs/latest/backends/ClickHouse/)
- [Dask](https://ibis-project.org/docs/latest/backends/Dask/)
- [DuckDB](https://ibis-project.org/docs/latest/backends/DuckDB/)
- [MySQL](https://ibis-project.org/docs/latest/backends/MySQL/)
- [Pandas](https://ibis-project.org/docs/latest/backends/Pandas/)
- [PostgreSQL](https://ibis-project.org/docs/latest/backends/PostgreSQL/)
- [PySpark](https://ibis-project.org/docs/latest/backends/PySpark/)
- [SQLite](https://ibis-project.org/docs/latest/backends/SQLite/)

Learn more about using the library at https://ibis-project.org.
144 changes: 0 additions & 144 deletions asv.conf.json
View file

This file was deleted.

283 changes: 0 additions & 283 deletions benchmarks/benchmarks.py
View file

This file was deleted.

20 changes: 0 additions & 20 deletions ci/.env
View file

This file was deleted.

839 changes: 555 additions & 284 deletions ci/datamgr.py
View file

Large diffs are not rendered by default.

347 changes: 0 additions & 347 deletions ci/impalamgr.py
View file

This file was deleted.

121 changes: 121 additions & 0 deletions ci/make_geography_db.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
#!/usr/bin/env python
#
# Create the geography database for the tutorial
#
# This script creates the SQLite `geography.db` database, used in the Ibis
# tutorials.
#
# The source of the `countries` table is
# [GeoNames](https://www.geonames.org/countries/).
#
# The source of the `gdp` table is
# [World Bank website](https://data.worldbank.org/indicator/NY.GDP.MKTP.CD).
#
# The source of the `independence` table has been obtained from
# [Wikipedia](https://en.wikipedia.org/wiki/List_of_national_independence_days).
import datetime
import tempfile
from pathlib import Path
from typing import Any, Mapping

import click
import requests
import sqlalchemy as sa
import toolz

SCHEMAS = {
"countries": [
("iso_alpha2", sa.TEXT),
("iso_alpha3", sa.TEXT),
("iso_numeric", sa.INT),
("fips", sa.TEXT),
("name", sa.TEXT),
("capital", sa.TEXT),
("area_km2", sa.REAL),
("population", sa.INT),
("continent", sa.TEXT),
],
"gdp": [
("country_code", sa.TEXT),
("year", sa.INT),
("value", sa.REAL),
],
"independence": [
("country_code", sa.TEXT),
("independence_date", sa.DATE),
("independence_from", sa.TEXT),
],
}

POST_PARSE_FUNCTIONS = {
"independence": lambda row: toolz.assoc(
row,
"independence_date",
datetime.datetime.strptime(
row["independence_date"],
"%Y-%m-%d",
).date(),
)
}


def make_geography_db(
data: Mapping[str, Any],
con: sa.engine.Engine,
) -> None:
metadata = sa.MetaData(bind=con)

with con.begin() as bind:
for table_name, schema in SCHEMAS.items():
table = sa.Table(
table_name,
metadata,
*(
sa.Column(col_name, col_type)
for col_name, col_type in schema
),
)
table_columns = table.c.keys()
post_parse = POST_PARSE_FUNCTIONS.get(table_name, toolz.identity)

table.drop(bind=bind, checkfirst=True)
table.create(bind=bind)
bind.execute(
table.insert().values(),
[
post_parse(dict(zip(table_columns, row)))
for row in data[table_name]
],
)


@click.command(
help="Create the geography SQLite database for the Ibis tutorial"
)
@click.option(
"-d",
"--output-directory",
default=Path(tempfile.gettempdir()),
type=click.Path(dir_okay=True, path_type=Path),
help="The directory to which the database will be output",
show_default=True,
)
@click.option(
"-u",
"--input-data-url",
default="https://storage.googleapis.com/ibis-tutorial-data/geography.json",
type=str,
help="The URL containing the data with which to populate the database",
)
def main(output_directory: Path, input_data_url: str) -> None:
response = requests.get(input_data_url)
response.raise_for_status()
input_data = response.json()
db_path = output_directory.joinpath("geography.db")
con = sa.create_engine(f"sqlite:///{db_path}")
make_geography_db(input_data, con)
click.echo(db_path)


if __name__ == "__main__":
main()
88 changes: 0 additions & 88 deletions ci/recipe/meta.yaml
View file

This file was deleted.

5 changes: 4 additions & 1 deletion ci/release/dry_run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@ npx --yes \
-p "@semantic-release/changelog" \
-p "@semantic-release/exec" \
-p "@semantic-release/git" \
-p "@google/semantic-release-replace-plugin" \
-p "conventional-changelog-conventionalcommits" \
semantic-release \
--ci \
--dry-run \
--preset conventionalcommits \
--plugins \
--analyze-commits "@semantic-release/commit-analyzer" \
--generate-notes "@semantic-release/release-notes-generator" \
--verify-conditions "@semantic-release/changelog,@semantic-release/exec,@semantic-release/git" \
--prepare "@semantic-release/changelog,@semantic-release/exec" \
--prepare "@semantic-release/changelog,@semantic-release/exec,@google/semantic-release-replace-plugin" \
--branches "$branch" \
--repository-url "file://$PWD"
2 changes: 2 additions & 0 deletions ci/release/run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ npx --yes \
-p "@semantic-release/github" \
-p "@semantic-release/exec" \
-p "@semantic-release/git" \
-p "@google/semantic-release-replace-plugin" \
-p "conventional-changelog-conventionalcommits" \
semantic-release --ci
30 changes: 0 additions & 30 deletions ci/run_tests.sh
View file

This file was deleted.

148 changes: 75 additions & 73 deletions ci/schema/clickhouse.sql
View file
Original file line number Diff line number Diff line change
@@ -1,84 +1,86 @@
CREATE TABLE diamonds (
`date` Date DEFAULT today(),
carat Float64,
cut String,
color String,
clarity String,
depth Float64,
`table` Float64,
price Int64,
x Float64,
y Float64,
z Float64
) ENGINE = MergeTree(date, (`carat`), 8192);
carat Nullable(Float64),
cut Nullable(String),
color Nullable(String),
clarity Nullable(String),
depth Nullable(Float64),
`table` Nullable(Float64),
price Nullable(Int64),
x Nullable(Float64),
y Nullable(Float64),
z Nullable(Float64)
) ENGINE = Memory;

CREATE TABLE batting (
`date` Date DEFAULT today(),
`playerID` String,
`yearID` Int64,
stint Int64,
`teamID` String,
`lgID` String,
`G` Int64,
`AB` Int64,
`R` Int64,
`H` Int64,
`X2B` Int64,
`X3B` Int64,
`HR` Int64,
`RBI` Int64,
`SB` Int64,
`CS` Int64,
`BB` Int64,
`SO` Int64,
`IBB` Int64,
`HBP` Int64,
`SH` Int64,
`SF` Int64,
`GIDP` Int64
) ENGINE = MergeTree(date, (`playerID`), 8192);
`playerID` Nullable(String),
`yearID` Nullable(Int64),
stint Nullable(Int64),
`teamID` Nullable(String),
`lgID` Nullable(String),
`G` Nullable(Int64),
`AB` Nullable(Int64),
`R` Nullable(Int64),
`H` Nullable(Int64),
`X2B` Nullable(Int64),
`X3B` Nullable(Int64),
`HR` Nullable(Int64),
`RBI` Nullable(Int64),
`SB` Nullable(Int64),
`CS` Nullable(Int64),
`BB` Nullable(Int64),
`SO` Nullable(Int64),
`IBB` Nullable(Int64),
`HBP` Nullable(Int64),
`SH` Nullable(Int64),
`SF` Nullable(Int64),
`GIDP` Nullable(Int64)
) ENGINE = Memory;

CREATE TABLE awards_players (
`date` Date DEFAULT today(),
`playerID` String,
`awardID` String,
`yearID` Int64,
`lgID` String,
tie String,
notes String
) ENGINE = MergeTree(date, (`playerID`), 8192);
`playerID` Nullable(String),
`awardID` Nullable(String),
`yearID` Nullable(Int64),
`lgID` Nullable(String),
tie Nullable(String),
notes Nullable(String)
) ENGINE = Memory;

CREATE TABLE functional_alltypes (
`date` Date DEFAULT toDate(timestamp_col),
`index` Int64,
`Unnamed: 0` Int64,
id Int32,
bool_col UInt8,
tinyint_col Int8,
smallint_col Int16,
int_col Int32,
bigint_col Int64,
float_col Float32,
double_col Float64,
date_string_col String,
string_col String,
timestamp_col DateTime,
year Int32,
month Int32
) ENGINE = MergeTree(date, (`index`), 8192);
`index` Nullable(Int64),
`Unnamed: 0` Nullable(Int64),
id Nullable(Int32),
bool_col Nullable(UInt8),
tinyint_col Nullable(Int8),
smallint_col Nullable(Int16),
int_col Nullable(Int32),
bigint_col Nullable(Int64),
float_col Nullable(Float32),
double_col Nullable(Float64),
date_string_col Nullable(String),
string_col Nullable(String),
timestamp_col Nullable(DateTime),
year Nullable(Int32),
month Nullable(Int32)
) ENGINE = Memory;

CREATE TABLE tzone (
`date` Date DEFAULT today(),
ts DateTime,
key String,
value Float64
) ENGINE = MergeTree(date, (key), 8192);
ts Nullable(DateTime),
key Nullable(String),
value Nullable(Float64)
) ENGINE = Memory;

CREATE TABLE IF NOT EXISTS array_types (
`date` Date DEFAULT today(),
x Array(Int64),
y Array(String),
z Array(Float64),
grouper String,
scalar_column Float64
) ENGINE = MergeTree(date, (scalar_column), 8192);
x Array(Nullable(Int64)),
y Array(Nullable(String)),
z Array(Nullable(Float64)),
grouper Nullable(String),
scalar_column Nullable(Float64)
) ENGINE = Memory;

INSERT INTO array_types VALUES
([1, 2, 3], ['a', 'b', 'c'], [1.0, 2.0, 3.0], 'a', 1.0),
([4, 5], ['d', 'e'], [4.0, 5.0], 'a', 2.0),
([6, NULL], ['f', NULL], [6.0, NULL], 'a', 3.0),
([NULL, 1, NULL], [NULL, 'a', NULL], [], 'b', 4.0),
([2, NULL, 3], ['b', NULL, 'c'], NULL, 'b', 5.0),
([4, NULL, NULL, 5], ['d', NULL, NULL, 'e'], [4.0, NULL, NULL, 5.0], 'c', 6.0);
90 changes: 90 additions & 0 deletions ci/schema/duckdb.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
DROP TABLE IF EXISTS diamonds CASCADE;

CREATE TABLE diamonds (
carat FLOAT,
cut TEXT,
color TEXT,
clarity TEXT,
depth FLOAT,
"table" FLOAT,
price BIGINT,
x FLOAT,
y FLOAT,
z FLOAT
);

DROP TABLE IF EXISTS batting CASCADE;

CREATE TABLE batting (
"playerID" TEXT,
"yearID" BIGINT,
stint BIGINT,
"teamID" TEXT,
"lgID" TEXT,
"G" BIGINT,
"AB" BIGINT,
"R" BIGINT,
"H" BIGINT,
"X2B" BIGINT,
"X3B" BIGINT,
"HR" BIGINT,
"RBI" BIGINT,
"SB" BIGINT,
"CS" BIGINT,
"BB" BIGINT,
"SO" BIGINT,
"IBB" BIGINT,
"HBP" BIGINT,
"SH" BIGINT,
"SF" BIGINT,
"GIDP" BIGINT
);

DROP TABLE IF EXISTS awards_players CASCADE;

CREATE TABLE awards_players (
"playerID" TEXT,
"awardID" TEXT,
"yearID" BIGINT,
"lgID" TEXT,
tie TEXT,
notes TEXT
);

DROP TABLE IF EXISTS functional_alltypes CASCADE;

CREATE TABLE functional_alltypes (
"index" BIGINT,
"Unnamed: 0" BIGINT,
id INTEGER,
bool_col BOOLEAN,
tinyint_col SMALLINT,
smallint_col SMALLINT,
int_col INTEGER,
bigint_col BIGINT,
float_col REAL,
double_col DOUBLE PRECISION,
date_string_col TEXT,
string_col TEXT,
timestamp_col TIMESTAMP WITHOUT TIME ZONE,
year INTEGER,
month INTEGER
);

DROP TABLE IF EXISTS array_types CASCADE;

CREATE TABLE IF NOT EXISTS array_types (
x BIGINT[],
y TEXT[],
z DOUBLE PRECISION[],
grouper TEXT,
scalar_column DOUBLE PRECISION
);

INSERT INTO array_types VALUES
([1, 2, 3], ['a', 'b', 'c'], [1.0, 2.0, 3.0], 'a', 1.0),
([4, 5], ['d', 'e'], [4.0, 5.0], 'a', 2.0),
([6, NULL], ['f', NULL], [6.0, NULL], 'a', 3.0),
([NULL, 1, NULL], [NULL, 'a', NULL], [], 'b', 4.0),
([2, NULL, 3], ['b', NULL, 'c'], NULL, 'b', 5.0),
([4, NULL, NULL, 5], ['d', NULL, NULL, 'e'], [4.0, NULL, NULL, 5.0], 'c', 6.0);
24 changes: 21 additions & 3 deletions ci/schema/postgresql.sql
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
DROP SEQUENCE IF EXISTS test_sequence;
CREATE SEQUENCE IF NOT EXISTS test_sequence;

CREATE EXTENSION IF NOT EXISTS postgis;
CREATE EXTENSION IF NOT EXISTS plpython3u;

DROP TABLE IF EXISTS diamonds CASCADE;

CREATE TABLE diamonds (
Expand Down Expand Up @@ -126,13 +132,10 @@ DROP TABLE IF EXISTS intervals CASCADE;

CREATE TABLE IF NOT EXISTS intervals (
-- enable year and month when relativedelta support lands
-- a INTERVAL YEAR,
-- b INTERVAL MONTH,
c INTERVAL DAY,
d INTERVAL HOUR,
e INTERVAL MINUTE,
f INTERVAL SECOND,
-- g INTERVAL YEAR TO MONTH,
h INTERVAL DAY TO HOUR,
i INTERVAL DAY TO MINUTE,
j INTERVAL DAY TO SECOND,
Expand Down Expand Up @@ -164,3 +167,18 @@ CREATE TABLE IF NOT EXISTS not_supported_intervals (
b INTERVAL YEAR,
g INTERVAL YEAR TO MONTH
);

DROP TABLE IF EXISTS geo CASCADE;

CREATE TABLE IF NOT EXISTS geo (
id BIGSERIAL PRIMARY KEY,
geo_point GEOMETRY(POINT),
geo_linestring GEOMETRY(LINESTRING),
geo_polygon GEOMETRY(POLYGON),
geo_multipolygon GEOMETRY(MULTIPOLYGON)
);

CREATE INDEX IF NOT EXISTS idx_geo_geo_linestring ON geo USING GIST (geo_linestring);
CREATE INDEX IF NOT EXISTS idx_geo_geo_multipolygon ON geo USING GIST (geo_multipolygon);
CREATE INDEX IF NOT EXISTS idx_geo_geo_point ON geo USING GIST (geo_point);
CREATE INDEX IF NOT EXISTS idx_geo_geo_polygon ON geo USING GIST (geo_polygon);
8 changes: 8 additions & 0 deletions ci/schema/sqlite.sql
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
DROP TABLE IF EXISTS functional_alltypes;

CREATE TABLE functional_alltypes (
"index" BIGINT,
"Unnamed: 0" BIGINT,
Expand All @@ -19,6 +21,8 @@ CREATE TABLE functional_alltypes (

CREATE INDEX ix_functional_alltypes_index ON "functional_alltypes" ("index");

DROP TABLE IF EXISTS awards_players;

CREATE TABLE awards_players (
"playerID" TEXT,
"awardID" TEXT,
Expand All @@ -28,6 +32,8 @@ CREATE TABLE awards_players (
notes TEXT
);

DROP TABLE IF EXISTS batting;

CREATE TABLE batting (
"playerID" TEXT,
"yearID" BIGINT,
Expand All @@ -53,6 +59,8 @@ CREATE TABLE batting (
"GIDP" BIGINT
);

DROP TABLE IF EXISTS diamonds;

CREATE TABLE diamonds (
carat FLOAT,
cut TEXT,
Expand Down
84 changes: 8 additions & 76 deletions ci/udf/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,82 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.

cmake_minimum_required(VERSION 2.6)
project(impala_test_udfs LANGUAGES CXX)
cmake_minimum_required(VERSION 3.22)
set(CMAKE_CXX_COMPILER clang++)

# where to put generated libraries
# where to put generated libraries and binaries
set(LIBRARY_OUTPUT_PATH "build")
# where to put generated binaries
set(EXECUTABLE_OUTPUT_PATH "build")

find_program(CLANG_EXECUTABLE clang++)

SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb")

# find boost headers and libs
if ($ENV{ENABLE_NATIVE_TOOLCHAIN})
message(STATUS "Found Impala Toolchain, enabling toolchain boost")
set(ENV{BOOST_ROOT} "$ENV{IMPALA_TOOLCHAIN}/boost-$ENV{BOOST_VERSION}")
set(Boost_USE_STATIC_LIBS NOT ${BUILD_SHARED_LIBS})
set(Boost_USE_STATIC_RUNTIME ON)
else()
set(Boost_DEBUG TRUE)
set(Boost_USE_MULTITHREADED ON)
endif()

find_package(Boost REQUIRED)
include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
set(LIBS ${LIBS} ${Boost_LIBRARIES})
message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
message(STATUS "Boost libraries: " ${Boost_LIBRARIES})

set(CLANG_INCLUDE_FLAGS)

if ($ENV{ENABLE_NATIVE_TOOLCHAIN})
set(IMPALA_TOOLCHAIN ON)

# Set the root directory for the toolchain
set(TOOLCHAIN_ROOT $ENV{IMPALA_TOOLCHAIN})

if($ENV{USE_SYSTEM_GCC} EQUAL "0")
# If Impala is built with the GCC from the toolchain, change compiler and link paths
set(GCC_ROOT $ENV{IMPALA_TOOLCHAIN}/gcc-$ENV{IMPALA_GCC_VERSION})
set(CMAKE_C_COMPILER ${GCC_ROOT}/bin/gcc)
set(CMAKE_CXX_COMPILER ${GCC_ROOT}/bin/g++)

# The rpath is needed to be able to run the binaries produced by the toolchain without
# specifying an LD_LIBRARY_PATH
set(TOOLCHAIN_LINK_FLAGS "-Wl,-rpath,${GCC_ROOT}/lib64")
set(TOOLCHAIN_LINK_FLAGS "${TOOLCHAIN_LINK_FLAGS} -L${GCC_ROOT}/lib64")
endif()

if(NOT $ENV{IMPALA_CXX_COMPILER} STREQUAL "default")
set(CMAKE_CXX_COMPILER $ENV{IMPALA_CXX_COMPILER})
endif()
endif()

message(STATUS "Setup toolchain link flags ${TOOLCHAIN_LINK_FLAGS}")

if (IMPALA_TOOLCHAIN)
# When the Toolchain is used we use LLVM 3.3 that was built in a different path that it
# is invoked from, and a GCC that resides in a different location. LVVM 3.3 relies on
# hard-coded path information about where to find the system headers and does not support
# specifying the -gcc-toolchain flag to dynamically provide this information. Because of
# these reasons we need to manually add the system c++ headers to the path when we
# compile the IR code with clang.
set(CLANG_BASE_FLAGS
"-I${GCC_ROOT}/include/c++/$ENV{IMPALA_GCC_VERSION}"
"-I${GCC_ROOT}/include/c++/$ENV{IMPALA_GCC_VERSION}/backward"
"-I${GCC_ROOT}/include/c++/$ENV{IMPALA_GCC_VERSION}/x86_64-unknown-linux-gnu")
set(CLANG_INCLUDE_FLAGS ${CLANG_BASE_FLAGS})
endif()

set(CLANG_INCLUDE_FLAGS
${CLANG_INCLUDE_FLAGS}
"-I${Boost_INCLUDE_DIRS}"
)

set(CLANG_IR_CXX_FLAGS "-emit-llvm" "-c" "-DIR_COMPILE" "-DNDEBUG"
"-DHAVE_INTTYPES_H" "-DHAVE_NETINET_IN_H" "-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG"
"-DHAVE_INTTYPES_H" "-DHAVE_NETINET_IN_H"
"-Wno-deprecated" "-Wno-c++11-extensions" "-O1")

# Function to generate rule to cross compile a source file to an IR module.
Expand All @@ -99,7 +33,7 @@ function(COMPILE_TO_IR SRC_FILE)
set(OUTPUT_FILE "build/${BASE_NAME}.ll")
add_custom_command(
OUTPUT ${OUTPUT_FILE}
COMMAND ${CLANG_EXECUTABLE} ${CLANG_INCLUDE_FLAGS} ${CLANG_IR_CXX_FLAGS} ${SRC_FILE} -o ${OUTPUT_FILE}
COMMAND ${CMAKE_CXX_COMPILER} ${CLANG_INCLUDE_FLAGS} ${CLANG_IR_CXX_FLAGS} ${SRC_FILE} -o ${OUTPUT_FILE}
DEPENDS ${SRC_FILE})
add_custom_target(${BASE_NAME}-ir ALL DEPENDS ${OUTPUT_FILE})
endfunction(COMPILE_TO_IR)
Expand All @@ -109,7 +43,5 @@ add_library(udfsample SHARED udf-sample.cc)
add_library(udasample SHARED uda-sample.cc)

# Custom targest to cross compile UDA/UDF to ir
if (CLANG_EXECUTABLE)
COMPILE_TO_IR(udf-sample.cc )
COMPILE_TO_IR(uda-sample.cc )
endif(CLANG_EXECUTABLE)
COMPILE_TO_IR(udf-sample.cc)
COMPILE_TO_IR(uda-sample.cc)
6 changes: 3 additions & 3 deletions ci/udf/lib/udf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@
#ifndef IMPALA_UDF_UDF_H
#define IMPALA_UDF_UDF_H

#include <assert.h>
#include <boost/cstdint.hpp>
#include <string.h>
#include <cassert>
#include <cstdint>
#include <cstring>

/// This is the only Impala header required to develop UDFs and UDAs. This header
/// contains the types that need to be used and the FunctionContext object. The context
Expand Down
6 changes: 3 additions & 3 deletions codecov.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
comment:
layout: "reach, diff, flags, files"
behavior: default
require_changes: false # if true: only post the comment if coverage changes
require_base: no # [yes :: must have a base report to post]
require_head: yes # [yes :: must have a head report to post]
require_changes: false # if true: only post the comment if coverage changes
require_base: no # [yes :: must have a base report to post]
require_head: yes # [yes :: must have a head report to post]
branches: null
351 changes: 0 additions & 351 deletions conda-lock/linux-64-3.7.lock
View file

This file was deleted.

423 changes: 225 additions & 198 deletions conda-lock/linux-64-3.8.lock
View file

Large diffs are not rendered by default.

426 changes: 226 additions & 200 deletions conda-lock/linux-64-3.9.lock
View file

Large diffs are not rendered by default.

330 changes: 0 additions & 330 deletions conda-lock/osx-64-3.7.lock
View file

This file was deleted.

418 changes: 225 additions & 193 deletions conda-lock/osx-64-3.8.lock
View file

Large diffs are not rendered by default.

417 changes: 224 additions & 193 deletions conda-lock/osx-64-3.9.lock
View file

Large diffs are not rendered by default.

326 changes: 0 additions & 326 deletions conda-lock/win-64-3.7.lock
View file

This file was deleted.

411 changes: 218 additions & 193 deletions conda-lock/win-64-3.8.lock
View file

Large diffs are not rendered by default.

410 changes: 218 additions & 192 deletions conda-lock/win-64-3.9.lock
View file

Large diffs are not rendered by default.

50 changes: 43 additions & 7 deletions default.nix
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,27 @@
{ python ? "3.9"
{ python ? "3.10"
, doCheck ? true
}:
let
pkgs = import ./nix;
drv =
{ poetry2nix, python }:
{ poetry2nix
, python
, lib
}:

let
backends = [
"dask"
"datafusion"
"duckdb"
"pandas"
"sqlite"
];

backendsString = lib.concatStringsSep " " backends;
buildInputs = with pkgs; [ gdal_2 graphviz-nox proj sqlite ];
checkInputs = buildInputs;
in
poetry2nix.mkPoetryApplication {
inherit python;

Expand All @@ -20,21 +36,41 @@ let
);

preConfigure = ''
rm -f setup.py
rm setup.py
'';

buildInputs = with pkgs; [ graphviz-nox ];
checkInputs = with pkgs; [ graphviz-nox ];
inherit buildInputs checkInputs;

preCheck = ''
set -euo pipefail
tempdir="$(mktemp -d)"
cp -r ${pkgs.ibisTestingData}/* "$tempdir"
find "$tempdir" -type f -exec chmod u+rw {} +
find "$tempdir" -type d -exec chmod u+rwx {} +
ln -s "$tempdir" ci/ibis-testing-data
for backend in ${backendsString}; do
python ci/datamgr.py load "$backend"
done
'';

checkPhase = ''
set -euo pipefail
runHook preCheck
pytest ibis/tests --numprocesses auto
pytest --numprocesses auto --dist loadgroup -m '${lib.concatStringsSep " or " backends} or core'
runHook postCheck
'';

inherit doCheck;

pythonImportsCheck = [ "ibis" ];
pythonImportsCheck = [ "ibis" ] ++ (map (backend: "ibis.backends.${backend}") backends);
};
in
pkgs.callPackage drv {
Expand Down
134 changes: 0 additions & 134 deletions dev/genrelease.py
View file

This file was deleted.

238 changes: 0 additions & 238 deletions dev/merge-pr.py
View file

This file was deleted.

2 changes: 1 addition & 1 deletion dev/poetry2setup
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env nix-shell
#!nix-shell --pure -p python3Packages.black -p python3Packages.tomli -p python3Packages.poetry-core -p bash -i bash
#!nix-shell --pure -p python3Packages.black -p python3Packages.poetry-core -p bash -i bash
# vim: filetype=sh

set -euo pipefail
Expand Down
Loading