.circleci/config.yml

@@ -4,7 +4,7 @@ version: 2
 base: &base
   machine:
     image: circleci/classic:latest
-    docker_layer_caching: true
+    docker_layer_caching: false
   working_directory: ~/ibis/ci
 
 
@@ -22,7 +22,7 @@ test: &test
           fi
     - run:
         name: Start Databases
-        command: docker-compose up -d --no-build mysql postgres impala clickhouse
+        command: docker-compose up -d --no-build mysql postgres impala clickhouse mapd kudu-tserver kudu-master
     - run:
         name: Show Running Containers
         command: docker ps
@@ -40,25 +40,41 @@ test: &test
         command: docker-compose run waiter
     - run:
         name: Load Test Datasets
-        command: docker-compose run ibis ci/load-data.sh
+        command: docker-compose run -e LOGLEVEL=DEBUG ibis ci/load-data.sh
     - run:
-        name: Make test reports directory
-        command: sudo mkdir -p /tmp/ibis/test-reports/pytest
+        name: Make artifacts directories
+        command: |
+          sudo mkdir -p /tmp/ibis/test-reports/pytest
+          mkdir -p /tmp/env
     - run:
         name: Show packages in conda environment
         command: docker-compose run ibis conda list
+    - run:
+        name: Capture the exact environment used
+        command: docker-compose run ibis conda env export > /tmp/env/env.yml
     - run:
         name: Execute Pytest
         command: |
-          docker-compose run ibis pytest -rsxX \
-            --doctest-modules \
-            --doctest-ignore-import-errors \
-            --junitxml=/tmp/test-reports/pytest/junit.xml \
-            --tb=short -m "not udf" ibis
+          docker-compose run -e PYTHONHASHSEED=$PYTHONHASHSEED ibis \
+            pytest ibis -m "not udf" \
+                        -ra \
+                        --numprocesses auto \
+                        --doctest-modules \
+                        --doctest-ignore-import-errors \
+                        --junitxml=/tmp/test-reports/pytest/junit.xml \
+                        --cov=ibis \
+                        --cov-report=xml:/tmp/test-reports/pytest-cov/coverage.xml
     - store_test_results:
         path: /tmp/ibis/test-reports
     - store_artifacts:
         path: /tmp/ibis/test-reports
+    - store_artifacts:
+        path: /tmp/env/env.yml
+    - run:
+        name: Upload Coverage
+        command: |
+          bash <(curl -s https://codecov.io/bash) \
+            -f /tmp/ibis/test-reports/pytest-cov/coverage.xml
 
 
 build: &build
@@ -73,11 +89,15 @@ build: &build
         name: List Docker Images
         command: docker images
     - run:
-        name: Build Conda Recipe
-        command: |
-          docker-compose run ibis conda build conda-recipes/ibis-framework \
-            --python "${PYTHON_VERSION}" \
-            --channel conda-forge
+        name: Clone, update and build conda-forge recipe
+        command: docker-compose run ibis ci/feedstock.py test
+    - run:
+        name: Permission the host for package deployment
+        command: sudo chown -R "${USER}":"${USER}" /tmp/ibis/packages
+    - store_artifacts:
+        path: /tmp/ibis/packages/linux-64
+    - store_artifacts:
+        path: /tmp/ibis/packages/noarch
 
 
 benchmark: &benchmark
@@ -104,7 +124,7 @@ docs: &docs
           - "8d:b0:36:4d:f6:75:f2:5f:00:01:a1:53:63:52:f0:e2"
     - run:
         name: Start Databases
-        command: docker-compose up -d --no-build mysql postgres impala clickhouse
+        command: docker-compose up -d --no-build mysql postgres impala clickhouse mapd kudu-master kudu-tserver
     - run:
         name: Show Running Containers
         command: docker ps
@@ -158,33 +178,26 @@ jobs:
   python27_test:
     <<: *test
     environment:
+      - PYTHONHASHSEED: ""
       - PYTHON_VERSION: 2.7
 
-  python34_test:
-    <<: *test
-    environment:
-      - PYTHON_VERSION: 3.4
-
   python35_test:
     <<: *test
     environment:
+      - PYTHONHASHSEED: 0
       - PYTHON_VERSION: 3.5
 
   python36_test:
     <<: *test
     environment:
+      - PYTHONHASHSEED: ""
       - PYTHON_VERSION: 3.6
 
   python27_conda_build:
     <<: *build
     environment:
       - PYTHON_VERSION: 2.7
 
-  python34_conda_build:
-    <<: *build
-    environment:
-      - PYTHON_VERSION: 3.4
-
   python35_conda_build:
     <<: *build
     environment:
@@ -211,30 +224,12 @@ workflows:
   test:
     jobs:
       - python27_test
-      - python34_test
-      - python35_test
-      - python36_test
+      - python27_conda_build
 
-      - python27_conda_build:
-          requires:
-            - python27_test
-
-      - python34_conda_build:
-          requires:
-            - python34_test
-
-      - python35_conda_build:
-          requires:
-            - python35_test
-
-      - python36_conda_build:
-          requires:
-            - python36_test
-
-      - python36_benchmark:
-          requires:
-            - python36_test
+      - python35_test
+      - python35_conda_build
 
-      - python36_docs:
-          requires:
-            - python36_test
+      - python36_test
+      - python36_conda_build
+      - python36_benchmark
+      - python36_docs

.coveragerc

@@ -1,8 +1,14 @@
 [run]
-branch: True
-omit: ibis/tests/*,
-      ibis/*/tests/*,
-      ibis/cloudpickle.py
-      ibis/tasks.py
-      ibis/server.py
-      ibis/wire.py
+branch = True
+source = ibis
+
+[report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+ignore_errors = True
+omit =
+    _version.py
+    tests/*

MANIFEST.in

@@ -4,7 +4,6 @@ include README.md
 include LICENSE.txt
 graft ibis
 graft LICENSES
-graft conda-recipes
 
 graft docs
 

Makefile

@@ -1,27 +1,61 @@
 .PHONY: all clean-pyc develop lint test docclean docs docserve
 
 SHELL := /bin/bash
+ENVKIND := docs
+MAKEFILE_DIR = $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+COMPOSE_FILE := "$(MAKEFILE_DIR)/ci/docker-compose.yml"
+DOCKER := ENVKIND=$(ENVKIND) docker-compose -f $(COMPOSE_FILE)
+DOCKER_RUN := $(DOCKER) run --rm
 
-all:
-	python setup.py build_ext --inplace
+clean:
+	@find . -name '*.pyc' -exec -delete
 
-clean-pyc:
-	find . -name "*.pyc" -exec rm -rf {} \;
-
-develop: clean-pyc
-	python setup.py develop
+develop: clean
+	@python setup.py develop
 
 lint:
-	flake8
+	@flake8
 
-test:
-	pytest --pyargs ibis -m 'not impala and not hdfs'
+stop:
+# stop all running docker compose services
+	@$(DOCKER) rm --force --stop
 
-docclean:
-	$(MAKE) -C docs clean
+build:
+# build the ibis image
+	@$(DOCKER) build --pull ibis
+
+start:
+# start all docker compose services
+	@$(DOCKER) up -d --no-build mapd postgres mysql clickhouse impala kudu-master kudu-tserver
+# wait for services to start
+	@$(DOCKER_RUN) waiter
+
+load:
+	@$(DOCKER_RUN) -e LOGLEVEL ibis ci/load-data.sh
 
-docs:
-	$(MAKE) -C docs html
+restart: stop
+	@$(MAKE) start
 
-docserve: docs
-	pushd docs/build/html && python -m http.server --bind localhost
+init: restart
+	@$(MAKE) build
+	@$(MAKE) load
+
+test:
+	@ENVKIND=$(ENVKIND) $(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not udf'
+
+testfast:
+	@ENVKIND=$(ENVKIND) $(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not udf and not impala and not bigquery'
+
+docclean:
+	@$(DOCKER_RUN) ibis rm -rf /tmp/docs.ibis-project.org
+
+docs: docclean
+	@$(DOCKER_RUN) ibis ping -c 1 quickstart.cloudera
+	@$(DOCKER_RUN) ibis git clone --branch gh-pages https://github.com/ibis-project/docs.ibis-project.org /tmp/docs.ibis-project.org
+	@$(DOCKER_RUN) ibis find /tmp/docs.ibis-project.org -maxdepth 1 ! -wholename /tmp/docs.ibis-project.org \
+	    ! -name '*.git' \
+	    ! -name '.' \
+	    ! -name 'CNAME' \
+	    ! -name '*.nojekyll' \
+	    -exec rm -rf {} \;
+	@$(DOCKER_RUN) ibis sphinx-build -b html docs/source /tmp/docs.ibis-project.org -W -j auto -T

README.md

@@ -1,9 +1,14 @@
 # Ibis: Python data analysis framework for Hadoop and SQL engines
 
-[![Anaconda-Server Badge](https://anaconda.org/conda-forge/ibis-framework/badges/version.svg)](https://anaconda.org/conda-forge/ibis-framework)
-[![Documentation Status](https://img.shields.io/badge/docs-docs.ibis--project.org-blue.svg)](http://docs.ibis-project.org)
-[![CircleCI Status](https://circleci.com/gh/ibis-project/ibis.svg?style=shield&circle-token=b84ff8383cbb0d6788ee0f9635441cb962949a4f)](https://circleci.com/gh/ibis-project/ibis/tree/master)
-[![AppVeyor Status](https://ci.appveyor.com/api/projects/status/github/ibis-project/ibis?branch=master&svg=true)](https://ci.appveyor.com/project/cpcloud/ibis-xh5g1)
+|Service|Status|
+| -------------: | :---- |
+| Documentation | [![Documentation Status](https://img.shields.io/badge/docs-docs.ibis--project.org-blue.svg)](http://docs.ibis-project.org) |
+| Conda packages | [![Anaconda-Server Badge](https://anaconda.org/conda-forge/ibis-framework/badges/version.svg)](https://anaconda.org/conda-forge/ibis-framework) |
+| PyPI | ![PyPI](https://img.shields.io/pypi/v/ibis-framework.svg) |
+| CircleCI | [![CircleCI Status](https://circleci.com/gh/ibis-project/ibis.svg?style=shield&circle-token=b84ff8383cbb0d6788ee0f9635441cb962949a4f)](https://circleci.com/gh/ibis-project/ibis/tree/master) |
+| AppVeyor | [![AppVeyor Status](https://ci.appveyor.com/api/projects/status/github/ibis-project/ibis?branch=master&svg=true)](https://ci.appveyor.com/project/cpcloud/ibis-xh5g1) |
+| Coverage | ![Codecov branch](https://img.shields.io/codecov/c/github/ibis-project/ibis/master.svg) |
+
 
 Ibis is a toolbox to bridge the gap between local Python environments, remote
 storage, execution systems like Hadoop components (HDFS, Impala, Hive, Spark)
@@ -33,6 +38,7 @@ Ibis currently provides tools for interacting with the following systems:
 - [Pandas](https://pandas.pydata.org/) [DataFrames](http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe) (Experimental)
 - [Clickhouse](https://clickhouse.yandex)
 - [BigQuery](https://cloud.google.com/bigquery)
+- [MapD](https://www.mapd.com/) (Experimental)
 
 Learn more about using the library at http://docs.ibis-project.org and read the
 project blog at http://ibis-project.org for news and updates.

appveyor.yml

@@ -10,7 +10,7 @@ environment:
   PGPASSWORD: "Password12!"
 
   IBIS_TEST_DOWNLOAD_DIRECTORY: "%USERPROFILE%"
-  IBIS_TEST_DOWNLOAD_BASE_URL: "https://storage.googleapis.com/ibis-ci-data"
+  IBIS_TEST_DOWNLOAD_BASE_URL: "https://storage.googleapis.com/ibis-testing-data"
   IBIS_TEST_DOWNLOAD_NAME: "ibis-testing-data.tar.gz"
 
   IBIS_TEST_DATA_DIRECTORY: "%USERPROFILE%\\ibis-testing-data"
@@ -34,7 +34,6 @@ environment:
 
   matrix:
     - PYTHON_VERSION: "2.7"
-    - PYTHON_VERSION: "3.4"
     - PYTHON_VERSION: "3.5"
     - PYTHON_VERSION: "3.6"
 
@@ -50,15 +49,13 @@ test_script:
   - "%CONDA% config --set always_yes true"
   - "%CONDA% create --name \"ibis_%PYTHON_VERSION%\" python=%PYTHON_VERSION% --channel conda-forge"
   - "%ACTIVATE% \"ibis_%PYTHON_VERSION%\""
-  - "%CONDA% install --channel conda-forge pytables numpy sqlalchemy psycopg2 graphviz click mock plumbum flake8 pytest"
+  - "%CONDA% install --channel conda-forge pytables numpy \"sqlalchemy<1.1.15\" psycopg2 graphviz click mock plumbum flake8 pytest pytest-xdist"
   - "%CONDA% list"
   - "pip install -e .\"[sqlite, postgres, mysql, visualization, pandas, csv, hdf5]\""
 
-  - "flake8"
-
   - "python ci\\datamgr.py download"
   - "python ci\\datamgr.py parquet -i"
   - "python ci\\datamgr.py mysql"
   - "python ci\\datamgr.py sqlite"
   - "python ci\\datamgr.py postgres"
-  - "pytest --tb=short -m \"not backend and not clickhouse and not impala and not hdfs and not bigquery\" -rs ibis"
+  - "pytest --tb=short -n auto -m \"not backend and not clickhouse and not impala and not hdfs and not bigquery and not mapd\" -ra ibis"

asv.conf.json

@@ -59,7 +59,9 @@
 	"six": [],
 	"multipledispatch": [],
 	"impyla": [],
-	"sqlalchemy": []
+	"sqlalchemy": [],
+	"pytz": [],
+	"regex": []
     },
 
     // Combinations of libraries/python versions can be excluded/included
@@ -88,12 +90,10 @@
     //     {"environment_type": "conda", "six": null}, // don't run without six on conda
     // ],
     //
-    // "include": [
-    //     // additional env for python2.7
-    //     {"python": "2.7", "numpy": "1.8"},
-    //     // additional env if run on windows+conda
-    //     {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""},
-    // ],
+    "include": [
+        // additional env for python2.7
+        {"python": "2.7", "funcsigs": [], "enum34": [], "functools32": []}
+    ],
 
     // The directory (relative to the current directory) that benchmarks are
     // stored in.  If not provided, defaults to "benchmarks"