.circleci/config.yml

@@ -28,7 +28,7 @@ test: &test
         command: docker ps
     - run:
         name: Build Ibis Image
-        command: docker-compose build --pull ibis
+        command: docker-compose build ibis
     - run:
         name: List Docker Images
         command: docker images
@@ -51,7 +51,7 @@ test: &test
         command: docker-compose run ibis conda list
     - run:
         name: Capture the exact environment used
-        command: docker-compose run ibis conda env export > /tmp/env/env.yml
+        command: docker-compose run ibis conda list --export > /tmp/env/env.yml
     - run:
         name: Execute Pytest
         command: |
@@ -84,7 +84,7 @@ build: &build
         path: ~/ibis
     - run:
         name: Build Ibis Image
-        command: docker-compose build --pull ibis
+        command: docker-compose build ibis
     - run:
         name: List Docker Images
         command: docker images
@@ -107,7 +107,7 @@ benchmark: &benchmark
         path: ~/ibis
     - run:
         name: Build Ibis Image
-        command: docker-compose build --pull ibis
+        command: docker-compose build ibis
     - run:
         name: Run Benchmark (ASV)
         command: |
@@ -130,28 +130,34 @@ docs: &docs
         command: docker ps
     - run:
         name: Build Ibis Documentation Image
-        command: docker-compose build --pull ibis
+        command: docker-compose build ibis && docker-compose build ibis-docs
     - run:
         name: List Docker Images
         command: docker images
     - run:
         name: Wait for Databases
         command: docker-compose run waiter
+    - run:
+        name: Show Running Containers
+        command: docker ps
     - run:
         name: Load Test Datasets
-        command: docker-compose run ibis ci/load-data.sh
+        command: docker-compose run ibis-docs ci/load-data.sh
     - run:
         name: Ping the Notebook Impala Hostname
-        command: docker-compose run ibis ping -c 1 quickstart.cloudera
+        command: docker-compose run ibis-docs ping -c 1 quickstart.cloudera
     - run:
         name: Clone Documentation Repository
-        command: docker-compose run ibis git clone --branch gh-pages https://github.com/ibis-project/docs.ibis-project.org /tmp/docs.ibis-project.org
+        command: docker-compose run ibis-docs git clone --branch gh-pages https://github.com/ibis-project/docs.ibis-project.org /tmp/docs.ibis-project.org
     - run:
         name: Clear Out Old Documentation Files
-        command: docker-compose run ibis find /tmp/docs.ibis-project.org -maxdepth 1 ! -name '*.git' ! -wholename /tmp/docs.ibis-project.org ! -name 'CNAME' ! -name '*.nojekyll' -exec rm -rf {} \;
+        command: docker-compose run ibis-docs find /tmp/docs.ibis-project.org -maxdepth 1 ! -name '*.git' ! -wholename /tmp/docs.ibis-project.org ! -name 'CNAME' ! -name '*.nojekyll' -exec rm -rf {} \;
+    - run:
+        name: Show the environment used to build the docs
+        command: docker-compose run ibis-docs conda list
     - run:
         name: Build Documentation
-        command: docker-compose run ibis sphinx-build -b html docs/source /tmp/docs.ibis-project.org -W -T -j auto
+        command: docker-compose run ibis-docs sphinx-build -b html docs/source /tmp/docs.ibis-project.org -W -T
     - run:
         name: Who is Running
         command: |
@@ -175,38 +181,27 @@ docs: &docs
         path: /tmp/ibis/docs.ibis-project.org
 
 jobs:
-  python27_test:
-    <<: *test
-    environment:
-      - PYTHONHASHSEED: ""
-      - PYTHON_VERSION: 2.7
-
-  python35_test:
-    <<: *test
-    environment:
-      - PYTHONHASHSEED: 0
-      - PYTHON_VERSION: 3.5
-
   python36_test:
     <<: *test
     environment:
       - PYTHONHASHSEED: ""
       - PYTHON_VERSION: 3.6
 
-  python27_conda_build:
-    <<: *build
+  python37_test:
+    <<: *test
     environment:
-      - PYTHON_VERSION: 2.7
+      - PYTHONHASHSEED: ""
+      - PYTHON_VERSION: 3.7
 
-  python35_conda_build:
+  python36_conda_build:
     <<: *build
     environment:
-      - PYTHON_VERSION: 3.5
+      - PYTHON_VERSION: 3.6
 
-  python36_conda_build:
+  python37_conda_build:
     <<: *build
     environment:
-      - PYTHON_VERSION: 3.6
+      - PYTHON_VERSION: 3.7
 
   python36_benchmark:
     <<: *benchmark
@@ -223,13 +218,9 @@ workflows:
   version: 2
   test:
     jobs:
-      - python27_test
-      - python27_conda_build
-
-      - python35_test
-      - python35_conda_build
-
       - python36_test
-      - python36_conda_build
-      - python36_benchmark
+      - python37_test
       - python36_docs
+      - python36_benchmark
+      #- python36_conda_build
+      #- python37_conda_build

.coveragerc

@@ -10,5 +10,6 @@ exclude_lines =
     if __name__ == .__main__.:
 ignore_errors = True
 omit =
-    _version.py
-    tests/*
+    *_version.py
+    *tests*
+    *__init__*

.dockerignore

@@ -1,3 +1,5 @@
 ci/udf/CMakeCache.txt
 ci/udf/CMakeFiles/
 ci/udf/Makefile
+**/*.pyc
+**/__pycache__

.gitignore

@@ -65,3 +65,4 @@ ibis_testing*
 .asv/
 .ipynb_checkpoints/
 .pytest_cache
+.mypy_cache

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: git://github.com/pre-commit/pre-commit-hooks
+    rev: v2.1.0
+    hooks:
+      - id: flake8
+        types:
+          - python
+        args: ['--config=setup.cfg']
+
+  - repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+      - id: black

Makefile

@@ -1,61 +1,82 @@
-.PHONY: all clean-pyc develop lint test docclean docs docserve
+.PHONY: all clean develop typecheck stop build start load restart init test testmost testfast testparams docclean doc
 
 SHELL := /bin/bash
-ENVKIND := docs
 MAKEFILE_DIR = $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 COMPOSE_FILE := "$(MAKEFILE_DIR)/ci/docker-compose.yml"
-DOCKER := ENVKIND=$(ENVKIND) docker-compose -f $(COMPOSE_FILE)
+DOCKER := docker-compose -f $(COMPOSE_FILE)
 DOCKER_RUN := $(DOCKER) run --rm
+PYTEST_OPTIONS :=
+SERVICES := mapd postgres mysql clickhouse impala kudu-master kudu-tserver
 
 clean:
-	@find . -name '*.pyc' -exec -delete
+	python setup.py clean
+	find $(MAKEFILE_DIR) -name '*.pyc' -type f -delete
+	rm -rf $(MAKEFILE_DIR)/build $(MAKEFILE_DIR)/dist $(find $(MAKEFILE_DIR) -name __pycache__ -type d)
 
 develop: clean
-	@python setup.py develop
+	python setup.py develop
+	pre-commit install
+
+typecheck:
+	@mypy --ignore-missing-imports $(MAKEFILE_DIR)/ibis
 
 lint:
-	@flake8
+	flake8
 
 stop:
 # stop all running docker compose services
-	@$(DOCKER) rm --force --stop
+	$(DOCKER) rm --force --stop ${SERVICES}
 
 build:
 # build the ibis image
-	@$(DOCKER) build --pull ibis
+	$(DOCKER) build --pull ibis
 
 start:
 # start all docker compose services
-	@$(DOCKER) up -d --no-build mapd postgres mysql clickhouse impala kudu-master kudu-tserver
+	$(DOCKER) up -d --no-build ${SERVICES}
 # wait for services to start
-	@$(DOCKER_RUN) waiter
+	$(DOCKER_RUN) waiter
 
 load:
-	@$(DOCKER_RUN) -e LOGLEVEL ibis ci/load-data.sh
+	$(DOCKER_RUN) -e LOGLEVEL ibis ci/load-data.sh
 
 restart: stop
-	@$(MAKE) start
+	$(MAKE) start
 
 init: restart
-	@$(MAKE) build
-	@$(MAKE) load
+	$(MAKE) build
+	$(MAKE) load
+
+testparallel:
+	$(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not udf' \
+	    --doctest-modules --doctest-ignore-import-errors ${PYTEST_OPTIONS}
 
 test:
-	@ENVKIND=$(ENVKIND) $(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not udf'
+	$(MAKEFILE_DIR)/ci/test.sh ${PYTEST_OPTIONS} \
+	    --doctest-modules --doctest-ignore-import-errors
+
+testmost:
+	$(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not (udf or impala or hdfs)' \
+	    --doctest-modules --doctest-ignore-import-errors ${PYTEST_OPTIONS}
 
 testfast:
-	@ENVKIND=$(ENVKIND) $(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not udf and not impala and not bigquery'
+	$(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not (udf or impala or hdfs or bigquery)' \
+	    --doctest-modules --doctest-ignore-import-errors ${PYTEST_OPTIONS}
 
 docclean:
-	@$(DOCKER_RUN) ibis rm -rf /tmp/docs.ibis-project.org
+	$(DOCKER_RUN) ibis-docs rm -rf /tmp/docs.ibis-project.org
+
+builddoc:
+# build the ibis-docs image
+	$(DOCKER) build ibis-docs
 
-docs: docclean
-	@$(DOCKER_RUN) ibis ping -c 1 quickstart.cloudera
-	@$(DOCKER_RUN) ibis git clone --branch gh-pages https://github.com/ibis-project/docs.ibis-project.org /tmp/docs.ibis-project.org
-	@$(DOCKER_RUN) ibis find /tmp/docs.ibis-project.org -maxdepth 1 ! -wholename /tmp/docs.ibis-project.org \
+doc: builddoc docclean
+	$(DOCKER_RUN) ibis-docs ping -c 1 quickstart.cloudera
+	$(DOCKER_RUN) ibis-docs git clone --branch gh-pages https://github.com/ibis-project/docs.ibis-project.org /tmp/docs.ibis-project.org --depth 1
+	$(DOCKER_RUN) ibis-docs find /tmp/docs.ibis-project.org -maxdepth 1 ! -wholename /tmp/docs.ibis-project.org \
 	    ! -name '*.git' \
 	    ! -name '.' \
 	    ! -name 'CNAME' \
 	    ! -name '*.nojekyll' \
 	    -exec rm -rf {} \;
-	@$(DOCKER_RUN) ibis sphinx-build -b html docs/source /tmp/docs.ibis-project.org -W -j auto -T
+	$(DOCKER_RUN) ibis-docs sphinx-build -b html docs/source /tmp/docs.ibis-project.org -W -T

README.md

@@ -2,12 +2,12 @@
 
 |Service|Status|
 | -------------: | :---- |
-| Documentation | [![Documentation Status](https://img.shields.io/badge/docs-docs.ibis--project.org-blue.svg)](http://docs.ibis-project.org) |
+| Documentation  | [![Documentation Status](https://img.shields.io/badge/docs-docs.ibis--project.org-blue.svg)](http://docs.ibis-project.org) |
 | Conda packages | [![Anaconda-Server Badge](https://anaconda.org/conda-forge/ibis-framework/badges/version.svg)](https://anaconda.org/conda-forge/ibis-framework) |
-| PyPI | ![PyPI](https://img.shields.io/pypi/v/ibis-framework.svg) |
-| CircleCI | [![CircleCI Status](https://circleci.com/gh/ibis-project/ibis.svg?style=shield&circle-token=b84ff8383cbb0d6788ee0f9635441cb962949a4f)](https://circleci.com/gh/ibis-project/ibis/tree/master) |
-| AppVeyor | [![AppVeyor Status](https://ci.appveyor.com/api/projects/status/github/ibis-project/ibis?branch=master&svg=true)](https://ci.appveyor.com/project/cpcloud/ibis-xh5g1) |
-| Coverage | ![Codecov branch](https://img.shields.io/codecov/c/github/ibis-project/ibis/master.svg) |
+| PyPI           | ![PyPI](https://img.shields.io/pypi/v/ibis-framework.svg) |
+| CircleCI       | [![CircleCI Status](https://circleci.com/gh/ibis-project/ibis.svg?style=shield&circle-token=b84ff8383cbb0d6788ee0f9635441cb962949a4f)](https://circleci.com/gh/ibis-project/ibis/tree/master) |
+| Azure          | [![Azure Status](https://dev.azure.com/ibis-project/ibis/_apis/build/status/ibis-project.ibis)](https://dev.azure.com/ibis-project/ibis/_build) |
+| Coverage       | ![Codecov branch](https://img.shields.io/codecov/c/github/ibis-project/ibis/master.svg) |
 
 
 Ibis is a toolbox to bridge the gap between local Python environments, remote
@@ -29,16 +29,15 @@ conda install ibis-framework -c conda-forge
 
 Ibis currently provides tools for interacting with the following systems:
 
-- [Apache Impala (incubating)](http://impala.io/)
-- [Apache Kudu](http://getkudu.io)
+- [Apache Impala](https://impala.apache.org/)
+- [Apache Kudu](https://kudu.apache.org/)
 - [Hadoop Distributed File System (HDFS)](https://hadoop.apache.org/)
 - [PostgreSQL](https://www.postgresql.org/)
 - [MySQL](https://www.mysql.com/) (Experimental)
 - [SQLite](https://www.sqlite.org/)
 - [Pandas](https://pandas.pydata.org/) [DataFrames](http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe) (Experimental)
 - [Clickhouse](https://clickhouse.yandex)
 - [BigQuery](https://cloud.google.com/bigquery)
-- [MapD](https://www.mapd.com/) (Experimental)
+- [MapD](https://www.mapd.com) (Experimental)
 
-Learn more about using the library at http://docs.ibis-project.org and read the
-project blog at http://ibis-project.org for news and updates.
+Learn more about using the library at http://docs.ibis-project.org.