.circleci/config.yml

@@ -0,0 +1,278 @@
+version: 2
+
+environment: &environment
+  - IBIS_TEST_POSTGRES_DB: ibis_testing
+  - IBIS_POSTGRES_USER: ubuntu
+  - IBIS_POSTGRES_PASS: ubuntu
+  - IBIS_TEST_CLICKHOUSE_DB: ibis_testing
+  - IBIS_CLICKHOUSE_USER: default
+  - IBIS_CLICKHOUSE_HOST: localhost
+  - IBIS_CLICKHOUSE_PASS: ''
+
+  - DATA_URL: https://storage.googleapis.com/ibis-ci-data
+
+  # The following environment variables are necessary to run impala tests
+  - IBIS_TEST_IMPALA_HOST: localhost
+  - IBIS_TEST_IMPALA_PORT: 21050
+  - IBIS_TEST_NN_HOST: localhost
+  - IBIS_TEST_WEBHDFS_PORT: 50070
+  - IBIS_TEST_WEBHDFS_USER: ubuntu
+
+python_test_steps: &python_test_steps
+  - checkout
+  - attach_workspace:
+      at: /tmp/workspace
+  - run: cat /tmp/workspace/envars.sh | tee -a $BASH_ENV
+  - run: echo 'export ENV_NAME=ibis_${PYTHON_VERSION}' | tee -a $BASH_ENV
+  - run: sudo apt-get install -qq clang libboost-dev
+  - run: |
+      conda env create --quiet \
+        --name "${ENV_NAME}" \
+        --file "ci/requirements-dev-${PYTHON_VERSION}.yml"
+  - run: echo 'source activate ${ENV_NAME}' | tee -a $BASH_ENV
+  - run: flake8
+  - run: |
+      if [ -n "${GCLOUD_SERVICE_KEY}" ]; then
+        [ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ] && exit 1
+        echo "${GCLOUD_SERVICE_KEY}" | base64 --decode --ignore-garbage > "${GOOGLE_APPLICATION_CREDENTIALS}"
+
+        # confirm that we can connect to the bigquery test project
+        python -c 'from google.cloud.bigquery import Client; import os; Client(os.environ["GOOGLE_BIGQUERY_PROJECT_ID"])'
+      fi
+  - run: python setup.py develop
+  - run: |
+      dockerize -wait tcp://localhost:5432 \
+                -wait tcp://localhost:21050 \
+                -wait tcp://localhost:50070 \
+                -wait tcp://localhost:9000 \
+                -timeout 1m
+  - run: ci/datamgr.py download --directory /tmp/workspace
+  - run: |
+      ci/datamgr.py sqlite \
+        --database "$IBIS_TEST_SQLITE_DB_PATH" \
+        --data-directory "$DATA_DIR" \
+        --script ci/sqlite_load.sql \
+        functional_alltypes diamonds awards_players batting
+
+  - run: |
+      ci/datamgr.py postgres \
+        --database "$IBIS_TEST_POSTGRES_DB" \
+        --data-directory "$DATA_DIR" \
+        --script ci/postgresql_load.sql \
+        functional_alltypes diamonds awards_players batting
+
+  - run: |
+      ci/datamgr.py clickhouse \
+        --database "$IBIS_TEST_POSTGRES_DB" \
+        --data-directory "$DATA_DIR" \
+        --script ci/clickhouse_load.sql \
+        functional_alltypes diamonds awards_players batting
+
+  - run: test_data_admin.py load --data --data-dir "$DATA_DIR"
+  - run: mkdir -p /tmp/reports
+  - run: |
+      pytest -rsxX \
+        --doctest-modules \
+        --doctest-ignore-import-errors \
+        --junitxml=/tmp/reports/junit.xml \
+        --tb=short ibis
+  - store_test_results:
+      path: /tmp/reports
+  - store_artifacts:
+      path: /tmp/reports
+
+conda_build_steps: &conda_build_steps
+  - checkout
+  - attach_workspace:
+      at: /tmp/workspace
+  - run: cat /tmp/workspace/envars.sh | tee -a $BASH_ENV
+  - run: conda install conda-build --yes --channel conda-forge
+  - run: |
+      conda build conda-recipes/ibis-framework \
+        --python "${PYTHON_VERSION}" \
+        --channel conda-forge
+  - run: mkdir -p /tmp/artifacts
+  - run: |
+      CONDA_BUILD_TARBALL=$(conda build conda-recipes/ibis-framework \
+                              --output \
+                              --python "${PYTHON_VERSION}" \
+                              --channel conda-forge)
+      mv "$CONDA_BUILD_TARBALL" /tmp/artifacts
+  - store_artifacts:
+      path: /tmp/artifacts
+
+jobs:
+  setup_envars:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+    steps:
+      - run: mkdir -p /tmp/workspace
+      - run: |
+          cat <<EOF | tee /tmp/workspace/envars.sh
+          export PATH="/tmp/workspace/miniconda/bin:$PATH"
+          export DATA_DIR=/tmp/workspace/ibis-testing-data
+          export IBIS_TEST_CRUNCHBASE_DB=/tmp/workspace/crunchbase.db
+          export IBIS_TEST_SQLITE_DB_PATH=/tmp/workspace/ibis_testing.db
+          export FUNCTIONAL_ALLTYPES_CSV="\$DATA_DIR/functional_alltypes.csv"
+          export DIAMONDS_CSV="\$DATA_DIR/diamonds.csv"
+          export BATTING_CSV="\$DATA_DIR/batting.csv"
+          export AWARDS_PLAYERS_CSV="\$DATA_DIR/awards_players.csv"
+          export GOOGLE_APPLICATION_CREDENTIALS="\$HOME/gcloud-service-key.json"
+          export GOOGLE_BIGQUERY_PROJECT_ID="ibis-gbq"
+          EOF
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - envars.sh
+
+  install_miniconda:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+    steps:
+      - run: mkdir -p /tmp/workspace
+      - run: curl -o $HOME/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+      - run: bash $HOME/miniconda.sh -b -p /tmp/workspace/miniconda
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - miniconda
+
+  ################# TESTS ##################
+
+  python27_test:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+        environment:
+          PYTHON_VERSION: 2.7
+      - image: cpcloud86/impala:java8-1
+      - image: yandex/clickhouse-server
+    steps: *python_test_steps
+
+  python34_test:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+        environment:
+          PYTHON_VERSION: 3.4
+      - image: cpcloud86/impala:java8-1
+      - image: yandex/clickhouse-server
+    steps: *python_test_steps
+
+  python35_test:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+        environment:
+          PYTHON_VERSION: 3.5
+      - image: cpcloud86/impala:java8-1
+      - image: yandex/clickhouse-server
+    steps: *python_test_steps
+
+  python36_test:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+        environment:
+          PYTHON_VERSION: 3.6
+      - image: cpcloud86/impala:java8-1
+      - image: yandex/clickhouse-server
+    steps: *python_test_steps
+
+  ################# CONDA BUILD ##################
+
+  python27_conda_build:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+        environment:
+          PYTHON_VERSION: 2.7
+    steps: *conda_build_steps
+
+  python34_conda_build:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+        environment:
+          PYTHON_VERSION: 3.4
+    steps: *conda_build_steps
+
+  python35_conda_build:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+        environment:
+          PYTHON_VERSION: 3.5
+    steps: *conda_build_steps
+
+  python36_conda_build:
+    environment: *environment
+    docker:
+      - image: circleci/python:latest
+        environment:
+          PYTHON_VERSION: 3.6
+    steps: *conda_build_steps
+
+  ################### BENCHMARKING #################
+  benchmark:
+    docker:
+      - image: circleci/python:latest
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/workspace
+      - run: cat /tmp/workspace/envars.sh | tee -a $BASH_ENV
+      - run: pip install asv
+      - run: ci/asvconfig.py | tee $HOME/.asv-machine.json
+      - run: git remote add upstream git://github.com/ibis-project/ibis.git
+      - run: git fetch upstream refs/heads/master
+      - run: asv continuous -f 1.5 -e upstream/master "${CIRCLE_SHA1}" || echo > /dev/null
+
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - setup_envars
+      - install_miniconda
+
+      - python27_test:
+          requires:
+            - setup_envars
+            - install_miniconda
+      - python34_test:
+          requires:
+            - setup_envars
+            - install_miniconda
+      - python35_test:
+          requires:
+            - setup_envars
+            - install_miniconda
+      - python36_test:
+          requires:
+            - setup_envars
+            - install_miniconda
+
+      - python27_conda_build:
+          requires:
+            - setup_envars
+            - install_miniconda
+      - python34_conda_build:
+          requires:
+            - setup_envars
+            - install_miniconda
+      - python35_conda_build:
+          requires:
+            - setup_envars
+            - install_miniconda
+      - python36_conda_build:
+          requires:
+            - setup_envars
+            - install_miniconda
+
+      - benchmark:
+          requires:
+            - setup_envars
+            - install_miniconda

.gitignore

@@ -10,6 +10,7 @@
 *.log
 *.swp
 *.pdb
+.idea
 
 # Compiled source
 *.a

Makefile

@@ -1,10 +1,27 @@
+.PHONY: all clean-pyc develop lint test docclean docs docserve
+
 SHELL := /bin/bash
 
 all:
 	python setup.py build_ext --inplace
 
-impala-test:
-	pushd scripts && python load_test_data.py --udf && popd
-
 clean-pyc:
 	find . -name "*.pyc" -exec rm -rf {} \;
+
+develop: clean-pyc
+	python setup.py develop
+
+lint:
+	flake8
+
+test:
+	pytest --pyargs ibis -m 'not impala and not hdfs'
+
+docclean:
+	$(MAKE) -C docs clean
+
+docs:
+	$(MAKE) -C docs html
+
+docserve: docs
+	pushd docs/build/html && python -m http.server --bind localhost

README.md

@@ -1,29 +1,31 @@
-[![circleci](https://circleci.com/gh/ibis-project/ibis.svg?style=shield&circle-token=b84ff8383cbb0d6788ee0f9635441cb962949a4f)](https://circleci.com/gh/ibis-project/ibis/tree/master)
-[![appveyor](https://ci.appveyor.com/api/projects/status/github/ibis-project/ibis?branch=master&svg=true)](https://ci.appveyor.com/project/cpcloud/ibis-xh5g1)
+[![CircleCI Status](https://circleci.com/gh/ibis-project/ibis.svg?style=shield&circle-token=b84ff8383cbb0d6788ee0f9635441cb962949a4f)](https://circleci.com/gh/ibis-project/ibis/tree/master)
+[![AppVeyor Status](https://ci.appveyor.com/api/projects/status/github/ibis-project/ibis?branch=master&svg=true)](https://ci.appveyor.com/project/cpcloud/ibis-xh5g1)
+[![Documentation Status](https://readthedocs.org/projects/ibis-project/badge/?version=latest)](http://ibis-project.readthedocs.io/en/latest/?badge=latest)
 
 Current release from Anaconda.org [![Anaconda-Server Badge](https://anaconda.org/conda-forge/ibis-framework/badges/version.svg)](https://anaconda.org/conda-forge/ibis-framework)
 
 
 # Ibis: Python data analysis framework for Hadoop and SQL engines
 
-Ibis is a toolbox to bridge the gap between local Python environments and
-remote storage and execution systems like Hadoop components (HDFS, Impala,
-Hive, Spark) and SQL databases (Postgres, etc.). Its goal is to simplify
-analytical workflows and make you more productive.
+Ibis is a toolbox to bridge the gap between local Python environments, 
+remote storage, execution systems like Hadoop components (HDFS, Impala,
+Hive, Spark) and SQL databases. Its goal is to simplify analytical 
+workflows and make you more productive.
 
 Install Ibis from PyPI with:
 
     $ pip install ibis-framework
 
-At this time, Ibis provides tools for the interacting with the following
+At this time, Ibis provides tools for interacting with the following
 systems:
 
 - [Apache Impala (incubating)](http://impala.io/)
 - [Apache Kudu](http://getkudu.io)
-- Hadoop Distributed File System (HDFS)
+- [Hadoop Distributed File System (HDFS)](https://hadoop.apache.org/)
 - PostgreSQL (Experimental)
 - SQLite
 - Direct execution of ibis expressions against pandas object (Experimental)
+- [Clickhouse](https://clickhouse.yandex)
 
 Learn more about using the library at http://docs.ibis-project.org and read the
 project blog at http://ibis-project.org for news and updates.