.coveragerc

@@ -0,0 +1,8 @@
+[run]
+branch: True
+omit: ibis/tests/*,
+      ibis/*/tests/*,
+      ibis/cloudpickle.py
+      ibis/tasks.py
+      ibis/server.py
+      ibis/wire.py

.gitignore

@@ -34,6 +34,7 @@ dist
 *.egg-info
 # coverage
 .coverage
+coverage.xml
 
 # OS generated files
 .directory

.landscape.yaml

@@ -0,0 +1,25 @@
+strictness: medium
+
+output-format: grouped
+
+test-warnings: true
+doc-warnings: false
+
+ignore-paths:
+    - dev/merge-pr.py
+    - ibis/wire.py
+
+ignore-patterns:
+    - ^build
+    - ^scripts
+
+pep8:
+    run: true
+    options:
+        max-line-length: 79
+
+mccabe:
+    run: false
+
+pylint:
+    run: false

Makefile

@@ -5,3 +5,6 @@ all:
 
 impala-test:
 	pushd scripts && python load_test_data.py --udf && popd
+
+clean-pyc:
+	find . -name "*.pyc" -exec rm -rf {} \;

README.md

@@ -1,17 +1,51 @@
-Ibis
-===
+[![codecov.io](http://codecov.io/github/cloudera/ibis/coverage.svg?branch=master)](http://codecov.io/github/cloudera/ibis?branch=master)
 
-Ibis is a Python data analysis library enabling a 100% Python user workflow on
-top of big data systems for maximum performance and scalability. It was
-co-created by the creator of pandas (github.com/pydata/pandas) and designed to
-have a familiar user interface for folks used to small data on single machines
-in Python.
+# Ibis: Python data analysis framework for Hadoop and SQL engines
 
-Ibis targets Impala (github.com/cloudera/impala) as a first class execution
-engine, but may grow to encompass other backend systems in the future.
+Install Ibis from PyPI with:
 
-Learn much more at http://ibis-project.org
+    $ pip install ibis-framework
 
-Install ibis from PyPI with:
+Ibis is a Python data analysis library with a handful of related goals:
 
-    $ pip install ibis-framework
+- Enable data analysts to translation analytics on SQL engines to
+  Python code instead of the SQL code.
+- Provide high level analytics APIs and workflow tools to accelerate
+  productivity.
+- Provide high performance extensions for the Impala MPP query engine to enable
+  high performance Python code to operate in a scalable Hadoop-like environment
+- Abstract away database-specific SQL differences
+- Integrate with the Python data ecosystem using the above tools
+
+At this item, Ibis supports the following SQL-based systems:
+
+- Impala (on HDFS)
+- SQLite
+
+Ibis is being designed and led by the creator of pandas
+(github.com/pydata/pandas) and is intended to have a familiar user interface
+for folks used to small data on single machines in Python.
+
+Architecturally, Ibis features:
+
+- A pandas-like domain specific language (DSL) designed specifically for
+  analytics, aka **Ibis expressions**, that enable composable, reusable
+  analytics on structured data. If you can express something with a SQL SELECT
+  query, you can write it with Ibis.
+- A translation system that targets multiple SQL systems
+- Tools for wrapping user-defined functions in Impala and eventually other SQL
+  engines
+
+SQL engine support near on the horizon:
+
+- PostgreSQL
+- Redshift
+- Vertica
+- Spark SQL
+- Presto
+- Hive
+- MySQL / MariaDB
+
+Read the project blog at http://blog.ibis-project.org.
+
+Learn much more at http://ibis-project.org.

conda-recipes/ibis-framework/meta.yaml

@@ -1,11 +1,11 @@
 package:
   name: ibis-framework
-  version: "0.3.0"
+  version: "0.4.0"
 
 source:
-  fn: ibis-framework-0.3.0.tar.gz
-  url: https://pypi.python.org/packages/source/i/ibis-framework/ibis-framework-0.3.0.tar.gz
-  md5: 5aafdf43711df3fa45df3e17f68e46d5
+  fn: ibis-framework-0.4.0.tar.gz
+  url: https://pypi.python.org/packages/source/i/ibis-framework/ibis-framework-0.4.0.tar.gz
+  md5: 87323b54e070a538912cbb8d8174854a
 
 requirements:
   build:
@@ -14,28 +14,42 @@ requirements:
     - pytest
     - numpy >=1.7.0
     - pandas >=0.12.0
-    - impyla >=0.9.1
+    - impyla >=0.10.0
     - psutil ==0.6.1
-    - hdfs >=1.4.0
+    - hdfs ==1.4.3
     - six
 
   run:
     - python
     - pytest
     - numpy >=1.7.0
     - pandas >=0.12.0
-    - impyla >=0.9.1
+    - impyla >=0.10.0
     - psutil ==0.6.1
-    - hdfs >=1.4.0
+    - hdfs ==1.4.3
     - six
 
 test:
   imports:
     - ibis
     - ibis.expr
     - ibis.expr.tests
+    - ibis.hive
+    - ibis.hive.tests
+    - ibis.impala
+    - ibis.impala.tests
+    - ibis.spark
+    - ibis.spark.tests
     - ibis.sql
+    - ibis.sql.presto
+    - ibis.sql.presto.tests
+    - ibis.sql.redshift
+    - ibis.sql.redshift.tests
+    - ibis.sql.sqlite
+    - ibis.sql.sqlite.tests
     - ibis.sql.tests
+    - ibis.sql.vertica
+    - ibis.sql.vertica.tests
     - ibis.tests
 
 about:

dev/merge-pr.py

@@ -22,19 +22,20 @@
 #
 # Lightly modified from version of this script in incubator-parquet-format
 
+from __future__ import print_function
+
 from requests.auth import HTTPBasicAuth
 import requests
 
-import json
 import os
+import six
 import subprocess
 import sys
-import urllib2
 import textwrap
 
 IBIS_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
 PROJECT_NAME = 'ibis'
-print "IBIS_HOME = " + IBIS_HOME
+print("IBIS_HOME = " + IBIS_HOME)
 
 # Remote name with the PR
 PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "upstream")
@@ -73,14 +74,14 @@ def get_json_no_auth(url):
 
 
 def fail(msg):
-    print msg
+    print(msg)
     clean_up()
     sys.exit(-1)
 
 
 def run_cmd(cmd):
     # py2.6 does not have subprocess.check_output
-    if isinstance(cmd, basestring):
+    if isinstance(cmd, six.string_types):
         cmd = cmd.split(' ')
 
     popenargs = [cmd]
@@ -107,13 +108,13 @@ def continue_maybe(prompt):
 
 
 def clean_up():
-    print "Restoring head pointer to %s" % original_head
+    print("Restoring head pointer to %s" % original_head)
     run_cmd("git checkout %s" % original_head)
 
     branches = run_cmd("git branch").replace(" ", "").split("\n")
 
     for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches):
-        print "Deleting local branch %s" % branch
+        print("Deleting local branch %s" % branch)
         run_cmd("git branch -D %s" % branch)
 
 
@@ -167,13 +168,15 @@ def merge_pr(pr_num, target_ref):
     for c in commits:
         merge_message_flags += ["-m", c]
 
-    run_cmd(['git', 'commit', '--author="%s"' % primary_author] + merge_message_flags)
+    run_cmd(['git', 'commit', '--author="%s"' % primary_author] +
+            merge_message_flags)
 
     continue_maybe("Merge complete (local ref %s). Push to %s?" % (
         target_branch_name, PUSH_REMOTE_NAME))
 
     try:
-        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, target_ref))
+        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name,
+                                       target_ref))
     except Exception as e:
         clean_up()
         fail("Exception while pushing: %s" % e)
@@ -190,17 +193,20 @@ def cherry_pick(pr_num, merge_hash, default_branch):
     if pick_ref == "":
         pick_ref = default_branch
 
-    pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, pick_ref.upper())
+    pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num,
+                                             pick_ref.upper())
 
-    run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))
+    run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref,
+                                    pick_branch_name))
     run_cmd("git checkout %s" % pick_branch_name)
     run_cmd("git cherry-pick -sx %s" % merge_hash)
 
     continue_maybe("Pick complete (local ref %s). Push to %s?" % (
         pick_branch_name, PUSH_REMOTE_NAME))
 
     try:
-        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref))
+        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name,
+                                       pick_ref))
     except Exception as e:
         clean_up()
         fail("Exception while pushing: %s" % e)
@@ -214,7 +220,8 @@ def cherry_pick(pr_num, merge_hash, default_branch):
 
 
 def fix_version_from_branch(branch, versions):
-    # Note: Assumes this is a sorted (newest->oldest) list of un-released versions
+    #  Note: Assumes this is a sorted (newest->oldest) list of un-released
+    #  versions
     if branch == "master":
         return versions[0]
     else:
@@ -223,7 +230,8 @@ def fix_version_from_branch(branch, versions):
 
 
 branches = get_json("%s/branches" % GITHUB_API_BASE)
-branch_names = filter(lambda x: x.startswith("branch-"), [x['name'] for x in branches])
+branch_names = filter(lambda x: x.startswith("branch-"),
+                      [x['name'] for x in branches])
 # Assumes branch names can be sorted lexicographically
 # latest_branch = sorted(branch_names, reverse=True)[0]
 
@@ -239,23 +247,25 @@ def fix_version_from_branch(branch, versions):
 pr_repo_desc = "%s/%s" % (user_login, base_ref)
 
 if pr["merged"] is True:
-    print "Pull request %s has already been merged, assuming you want to backport" % pr_num
+    print("Pull request {0} has already been merged, assuming "
+          "you want to backport".format(pr_num))
     merge_commit_desc = run_cmd([
         'git', 'log', '--merges', '--first-parent',
         '--grep=pull request #%s' % pr_num, '--oneline']).split("\n")[0]
     if merge_commit_desc == "":
-        fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num)
+        fail("Couldn't find any merge commit for #{0}"
+             ", you may need to update HEAD.".format(pr_num))
 
     merge_hash = merge_commit_desc[:7]
     message = merge_commit_desc[8:]
 
-    print "Found: %s" % message
+    print("Found: %s" % message)
     maybe_cherry_pick(pr_num, merge_hash, latest_branch)
     sys.exit(0)
 
 if not bool(pr["mergeable"]):
-    msg = "Pull request %s is not mergeable in its current form.\n" % pr_num + \
-        "Continue? (experts only!)"
+    msg = ("Pull request {0} is not mergeable in its current form.\n"
+           "Continue? (experts only!)".format(pr_num))
     continue_maybe(msg)
 
 print ("\n=== Pull Request #%s ===" % pr_num)
@@ -269,4 +279,5 @@ def fix_version_from_branch(branch, versions):
 
 pick_prompt = "Would you like to pick %s into another branch?" % merge_hash
 while raw_input("\n%s (y/n): " % pick_prompt).lower() == "y":
-    merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)]
+    merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash,
+                                             latest_branch)]