.gitattributes

@@ -0,0 +1 @@
+ibis/_version.py export-subst

.gitignore

@@ -49,4 +49,9 @@ docs/source/generated-notebooks
 # UDF testing generated files
 testing/udf/CMakeCache.txt
 testing/udf/CMakeFiles/
-testing/udf/Makefile
+testing/udf/Makefile
+.cache/
+
+# test data
+scripts/ibis-testing*
+ibis_testing*

MANIFEST.in

@@ -25,3 +25,5 @@ global-exclude \#*
 global-exclude .git*
 global-exclude .DS_Store
 global-exclude *.png
+include versioneer.py
+include ibis/_version.py

README.md

@@ -8,16 +8,16 @@ Install Ibis from PyPI with:
 
 Ibis is a Python data analysis library with a handful of related goals:
 
-- Enable data analysts to translation analytics on SQL engines to
-  Python code instead of the SQL code.
+- Enable data analysts to transition analytics on SQL engines to Python code
+  instead of SQL code.
 - Provide high level analytics APIs and workflow tools to accelerate
   productivity.
 - Provide high performance extensions for the Impala MPP query engine to enable
   high performance Python code to operate in a scalable Hadoop-like environment
 - Abstract away database-specific SQL differences
 - Integrate with the Python data ecosystem using the above tools
 
-At this item, Ibis supports the following SQL-based systems:
+At this time, Ibis supports the following SQL-based systems:
 
 - Impala (on HDFS)
 - SQLite

conda-recipes/hdfs/meta.yaml

@@ -1,39 +1,42 @@
 package:
   name: hdfs
-  version: "1.4.3"
+  version: "2.0.2"
 
 source:
-  fn: hdfs-1.4.3.tar.gz
-  url: https://pypi.python.org/packages/source/h/hdfs/hdfs-1.4.3.tar.gz
-  md5: 872034eb169d9315b71eb00cb7e28ead
+  fn: hdfs-2.0.2.tar.gz
+  url: https://pypi.python.org/packages/source/h/hdfs/hdfs-2.0.2.tar.gz
+  md5: ade3a92382e0889e2a845b9cc707c704
 
 build:
   entry_points:
-    - hdfs = hdfs.__main__:main
-    # - hdfs-avro = hdfs.ext.avro:main  # disabled
+    - hdfscli = hdfs.__main__:main
+    # - hdfscli-avro = hdfs.ext.avro.__main__:main  # disabled
 
 requirements:
   build:
     - python
     - setuptools
     - docopt
-    - requests >=2.0.1
+    - requests >=2.7.0
+    - six >=1.9.0
 
   run:
     - python
     - docopt
-    - requests >=2.0.1
+    - requests >=2.7.0
+    - six >=1.9.0
 
 test:
   imports:
     - hdfs
     - hdfs.ext
+    # - hdfs.ext.avro  # disabled
 
   commands:
-    - hdfs --help
-    # - hdfs-avro --help  # disabled
+    - hdfscli --help
+    # - hdfscli-avro --help  # disabled
 
 about:
   home: http://hdfscli.readthedocs.org
   license: MIT License
-  summary: 'HdfsCLI: a command line interface for WebHDFS.'
+  summary: 'HdfsCLI: API and command line interface for HDFS.'

conda-recipes/ibis-framework/bld.bat

@@ -1,6 +1,8 @@
-"%PYTHON%" setup.py install --single-version-externally-managed --record=record.txt
+"%PYTHON%" setup.py install
 if errorlevel 1 exit 1
 
+"%PYTHON%" -c "import ibis; print(ibis.__version__)" > __conda_version__.txt
+
 :: Add more build steps here, if they are necessary.
 
 :: See

conda-recipes/ibis-framework/build.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-$PYTHON setup.py install --single-version-externally-managed --record=record.txt
+$PYTHON setup.py install
+
+$PYTHON -c "import ibis; print(ibis.__version__)" > __conda_version__.txt
 
 # Add more build steps here, if they are necessary.
 

conda-recipes/ibis-framework/meta.yaml

@@ -1,11 +1,9 @@
 package:
   name: ibis-framework
-  version: "0.4.0"
+  version: "SETBYINSTALL"
 
 source:
-  fn: ibis-framework-0.4.0.tar.gz
-  url: https://pypi.python.org/packages/source/i/ibis-framework/ibis-framework-0.4.0.tar.gz
-  md5: 87323b54e070a538912cbb8d8174854a
+  path: ../..
 
 requirements:
   build:
@@ -15,8 +13,8 @@ requirements:
     - numpy >=1.7.0
     - pandas >=0.12.0
     - impyla >=0.10.0
-    - psutil ==0.6.1
-    - hdfs ==1.4.3
+    - hdfs >=2.0.0
+    - sqlalchemy >=1.0.0
     - six
 
   run:
@@ -25,8 +23,8 @@ requirements:
     - numpy >=1.7.0
     - pandas >=0.12.0
     - impyla >=0.10.0
-    - psutil ==0.6.1
-    - hdfs ==1.4.3
+    - hdfs >=2.0.0
+    - sqlalchemy >=1.0.0
     - six
 
 test:

conda-recipes/impyla/meta.yaml

@@ -1,20 +1,21 @@
 package:
   name: impyla
-  version: "0.10.0"
+  version: "0.11.2"
 
 source:
-  fn: impyla-0.10.0.tar.gz
-  url: https://pypi.python.org/packages/source/i/impyla/impyla-0.10.0.tar.gz
-  md5: 52c35b16b3897dbb53fb59510c33cfaa
+  git_url: https://github.com/cloudera/impyla
 
 build:
-  preserve_egg_dir: True
+  number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }}
+  string: py{{ environ.get('PY_VER').replace('.', '') }}_{{ environ.get('GIT_BUILD_STR', 'GIT_STUB') }}
 
 requirements:
   build:
     - python
     - setuptools
     - six
+    - thrift_sasl
+    - bitarray
     - thrift
     - llvmpy
     - numba
@@ -23,14 +24,15 @@ requirements:
     - python
     - setuptools
     - six
+    - thrift_sasl
+    - bitarray
     - thrift
     - llvmpy
     - numba
 
 test:
   imports:
     - impala
-    - impala._rpc
     - impala._thrift_api
     - impala._thrift_gen
     - impala._thrift_gen.ExecStats
@@ -41,7 +43,6 @@ test:
     - impala._thrift_gen.beeswax
     - impala._thrift_gen.fb303
     - impala._thrift_gen.hive_metastore
-    - impala.dbapi
     - impala.tests
     - impala.thrift
     # - impala.udf  # Test fails on osx-64

conda-recipes/thrift_sasl/bld.bat

@@ -0,0 +1,8 @@
+"%PYTHON%" setup.py install
+if errorlevel 1 exit 1
+
+:: Add more build steps here, if they are necessary.
+
+:: See
+:: http://docs.continuum.io/conda/build.html
+:: for a list of environment variables that are set during the build process.

conda-recipes/thrift_sasl/build.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+$PYTHON setup.py install
+
+# Add more build steps here, if they are necessary.
+
+# See
+# http://docs.continuum.io/conda/build.html
+# for a list of environment variables that are set during the build process.

conda-recipes/thrift_sasl/meta.yaml

@@ -0,0 +1,27 @@
+package:
+  name: thrift_sasl
+  version: "0.1.0"
+
+source:
+  fn: thrift_sasl-0.1.0.tar.gz
+  url: https://pypi.python.org/packages/source/t/thrift_sasl/thrift_sasl-0.1.0.tar.gz
+  md5: 0710ffa4ed33a657090a8305fd71ca1e
+
+requirements:
+  build:
+    - python
+    - setuptools
+    - thrift
+
+  run:
+    - python
+    - thrift
+
+test:
+  imports:
+    - thrift_sasl
+
+about:
+  home: https://github.com/cloudera/thrift_sasl
+  license: Apache License, Version 2.0
+  summary: 'Thrift SASL Python module that implements SASL transports for Thrift (`TSaslClientTransport`).'

docs/source/api.rst

@@ -50,12 +50,17 @@ Database methods
 .. autosummary::
    :toctree: generated/
 
-   Database.drop
-   Database.namespace
-   Database.table
+   ImpalaDatabase.create_table
+   ImpalaDatabase.drop
+   ImpalaDatabase.namespace
+   ImpalaDatabase.table
 
 Table methods
 ~~~~~~~~~~~~~
+
+The ``ImpalaClient`` object itself has many helper utility methods. You'll find
+the most methods on ``ImpalaTable``.
+
 .. autosummary::
    :toctree: generated/
 
@@ -71,14 +76,36 @@ Table methods
    ImpalaClient.truncate_table
    ImpalaClient.get_schema
    ImpalaClient.cache_table
+   ImpalaClient.load_data
+   ImpalaClient.get_options
+   ImpalaClient.set_options
+   ImpalaClient.set_compression_codec
+
+
+The best way to interact with a single table is through the ``ImpalaTable``
+object you get back from ``ImpalaClient.table``.
 
 .. autosummary::
    :toctree: generated/
 
+   ImpalaTable.add_partition
+   ImpalaTable.alter
+   ImpalaTable.alter_partition
+   ImpalaTable.column_stats
    ImpalaTable.compute_stats
+   ImpalaTable.describe_formatted
    ImpalaTable.drop
+   ImpalaTable.drop_partition
+   ImpalaTable.files
    ImpalaTable.insert
+   ImpalaTable.invalidate_metadata
+   ImpalaTable.load_data
+   ImpalaTable.metadata
+   ImpalaTable.partition_schema
+   ImpalaTable.partitions
+   ImpalaTable.refresh
    ImpalaTable.rename
+   ImpalaTable.stats
 
 Creating views is also possible:
 
@@ -263,6 +290,7 @@ Scalar or array methods
 
    ValueExpr.between
    ValueExpr.cast
+   ValueExpr.coalesce
    ValueExpr.fillna
    ValueExpr.isin
    ValueExpr.notin
@@ -281,14 +309,16 @@ Scalar or array methods
    ValueExpr.rdiv
    ValueExpr.rsub
 
+   ValueExpr.case
+   ValueExpr.cases
+   ValueExpr.substitute
+
 Array methods
 ~~~~~~~~~~~~~
 
 .. autosummary::
    :toctree: generated/
 
-   ArrayExpr.case
-   ArrayExpr.cases
    ArrayExpr.distinct
 
    ArrayExpr.count
@@ -398,6 +428,7 @@ All string operations are valid either on scalar or array values
    StringValue.translate
    StringValue.find_in_set
    StringValue.join
+   StringValue.replace
    StringValue.lpad
    StringValue.rpad
 
@@ -406,6 +437,7 @@ All string operations are valid either on scalar or array values
    StringValue.re_extract
    StringValue.re_replace
 
+.. _api.timestamp:
 
 Timestamp methods
 -----------------

docs/source/conf.py

@@ -19,7 +19,8 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+
+sys.path.insert(0, os.path.abspath('../sphinxext'))
 
 # -- General configuration ------------------------------------------------
 
@@ -33,7 +34,10 @@
     'sphinx.ext.autodoc',
     'sphinx.ext.mathjax',
     'sphinx.ext.autosummary',
-    'numpydoc'
+    'numpydoc',
+
+    'ipython_sphinxext.ipython_directive',
+    'ipython_sphinxext.ipython_console_highlighting',
 ]
 
 autosummary_generate = glob.glob("*.rst")