Merge pull request #1547 from kwmsmith/feature/py35

Python 3.5 support

.travis.yml

@@ -13,18 +13,25 @@ matrix:
       env: SPARK_VERSION=1.4
     - python: 3.4
       env: SPARK_VERSION=1.5
+    - python: 3.5
+      env: SPARK_VERSION=1.5
     - python: 2.7
       env: PANDAS_VERSION="git+https://github.com/pydata/pandas" SPARK_VERSION=1.5
     - python: 3.4
       env: PANDAS_VERSION="git+https://github.com/pydata/pandas" SPARK_VERSION=1.5
+    - python: 3.5
+      env: PANDAS_VERSION="git+https://github.com/pydata/pandas" SPARK_VERSION=1.5
   allow_failures:
     - python: 2.7
       env: PANDAS_VERSION="git+https://github.com/pydata/pandas" SPARK_VERSION=1.5
     - python: 3.4
       env: PANDAS_VERSION="git+https://github.com/pydata/pandas" SPARK_VERSION=1.5
+    - python: 3.5
+      env: PANDAS_VERSION="git+https://github.com/pydata/pandas" SPARK_VERSION=1.5
 
 services:
     - mongodb
+    - mysql
 
 addons:
   postgresql: "9.3"
@@ -39,11 +46,11 @@ install:
   - conda update conda
 
   # Install dependencies
-  - conda create -n blaze -c blaze python=$TRAVIS_PYTHON_VERSION pytest numpy sqlalchemy pandas h5py pip flask requests pytables cython bcolz xlrd coverage psutil networkx numba pyyaml cytoolz toolz multipledispatch dask pymongo=2.8 flask-cors psycopg2
+  - conda create -n blaze -c blaze python=$TRAVIS_PYTHON_VERSION pytest numpy sqlalchemy pandas h5py pip flask requests pytables cython bcolz xlrd coverage psutil networkx numba pyyaml cytoolz toolz multipledispatch dask pymongo=3.0 flask-cors psycopg2 pymysql
   - if [ -n "$PANDAS_VERSION" ]; then conda remove -n blaze pandas; fi
   - source activate blaze
 
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' || $TRAVIS_PYTHON_VERSION == '3.4' ]]; then conda install spark=$SPARK_VERSION -c blaze -c anaconda-cluster; fi
+  - conda install spark=$SPARK_VERSION -c blaze -c anaconda-cluster
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then conda install pyhive -c blaze; fi
 
   # blaze required deps
@@ -58,13 +65,14 @@ install:
   - python setup.py install
 
 before_script:
-  - sleep 15
   - "mongo admin --eval 'db.runCommand({setParameter: 1, textSearchEnabled: true});'"
   - psql -c "create database test;" -U postgres
+  - mysql -e "create database if not exists test;" -u root
+  - sleep 15
 
 script:
   - echo '[pytest]' > pytest.ini
-  - echo 'addopts = -vv -r sxX --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' --pyargs blaze docs' >> pytest.ini
+  - echo "addopts = -vv -r sxX --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' --pyargs blaze docs" >> pytest.ini
   - echo 'norecursedirs = docs/source/scripts' >> pytest.ini
   - coverage run --include='blaze/*' --omit='blaze/_version.py' $(which py.test)
   - coverage report --show-missing

blaze/compute/tests/test_hdfstore.py

@@ -1,65 +1,52 @@
 import pytest
 tables = pytest.importorskip('tables')
 
-from blaze.compute.hdfstore import *
 from blaze.utils import tmpfile
-from datashape import discover, dshape
-from blaze import symbol, compute, data
+from blaze import symbol, discover, compute, pre_compute
 import pandas as pd
 from datetime import datetime
-from odo import Chunks, into
-import os
+from odo import Chunks, odo
 
 
-try:
-    f = pd.HDFStore('foo')
-except (RuntimeError, ImportError) as e:
-    pytest.skip('skipping test_hdfstore.py %s' % e)
-else:
-    f.close()
-    os.remove('foo')
+@pytest.fixture
+def df():
+    return pd.DataFrame([['a', 1, 10., datetime(2000, 1, 1)],
+                         ['ab', 2, 20., datetime(2000, 2, 2)],
+                         ['abc', 3, 30., datetime(2000, 3, 3)],
+                         ['abcd', 4, 40., datetime(2000, 4, 4)]],
+                        columns=['name', 'a', 'b', 'time'])
 
 
-df = pd.DataFrame([['a', 1, 10., datetime(2000, 1, 1)],
-                   ['ab', 2, 20., datetime(2000, 2, 2)],
-                   ['abc', 3, 30., datetime(2000, 3, 3)],
-                   ['abcd', 4, 40., datetime(2000, 4, 4)]],
-                   columns=['name', 'a', 'b', 'time'])
+@pytest.fixture
+def s(hdf):
+    return symbol('s', discover(hdf))
 
 
-def test_hdfstore():
+@pytest.yield_fixture(params=['fixed', 'table'])
+def hdf(df, request):
     with tmpfile('.hdf5') as fn:
-        df.to_hdf(fn, '/appendable', format='table')
-        df.to_hdf(fn, '/fixed')
+        df.to_hdf(fn, '/data', format=request.param)
+        df.to_hdf(fn, '/nested/data', format=request.param)
+        with pd.HDFStore(fn, mode='r') as r:
+            yield r
 
-        hdf = data('hdfstore://%s' % fn)
-        s = symbol('s', discover(hdf))
 
-        assert isinstance(compute(s.fixed, hdf),
-                          (pd.DataFrame, pd.io.pytables.Fixed))
-        assert isinstance(compute(s.appendable, hdf),
-                          (pd.io.pytables.AppendableFrameTable, Chunks))
+def test_basic_compute(hdf, s):
+    result = compute(s.data, hdf)
+    types = (
+        pd.DataFrame,
+        pd.io.pytables.Fixed,
+        pd.io.pytables.AppendableFrameTable,
+        Chunks
+    )
+    assert isinstance(result, types)
 
-        s = symbol('s', discover(df))
-        f = data('hdfstore://%s::/fixed' % fn)
-        a = data('hdfstore://%s::/appendable' % fn)
-        assert isinstance(pre_compute(s, a), Chunks)
 
-        hdf.data.close()
-        f.data.parent.close()
-        a.data.parent.close()
+def test_pre_compute(hdf, s):
+    result = pre_compute(s, hdf.get_storer('data'))
+    assert isinstance(result, (pd.DataFrame, Chunks))
 
 
-
-def test_groups():
-    with tmpfile('.hdf5') as fn:
-        df.to_hdf(fn, '/data/fixed')
-
-        hdf = data('hdfstore://%s' % fn)
-        assert dshape(discover(hdf)) == dshape(discover({'data': {'fixed': df}}))
-
-        s = symbol('s', discover(hdf))
-
-        assert list(compute(s.data.fixed, hdf).a) == [1, 2, 3, 4]
-
-        hdf.data.close()
+def test_groups(hdf, df, s):
+    assert discover(hdf) == discover(dict(data=df, nested=dict(data=df)))
+    assert odo(compute(s.nested.data.a, hdf), list) == [1, 2, 3, 4]

blaze/compute/tests/test_mongo_compute.py

@@ -1,25 +1,34 @@
 from __future__ import absolute_import, division, print_function
 
 import pytest
-import platform
 pymongo = pytest.importorskip('pymongo')
 
 from datetime import datetime
 from toolz import pluck, reduceby, groupby
 
 from datashape import Record
-from blaze import into, compute, compute_up, discover, dshape, data
+import blaze
+from blaze import compute, compute_up, discover, dshape, data
 
 from blaze.compute.mongo import MongoQuery
 from blaze.expr import symbol, by, floor, ceil
 from blaze.compatibility import xfail
 
+
+def into(*args, **kwargs):
+    try:
+        return blaze.into(*args, **kwargs)
+    except pymongo.errors.ConnectionFailure:
+        pytest.skip('No mongo server running')
+
+
 @pytest.fixture(scope='module')
 def mongo_host_port():
     import os
     return (os.environ.get('MONGO_IP', 'localhost'),
             os.environ.get('MONGO_PORT', 27017))
 
+
 @pytest.fixture(scope='module')
 def conn(mongo_host_port):
     host, port = mongo_host_port
@@ -343,7 +352,8 @@ def test_missing_values(missing_vals):
 
 def test_datetime_access(date_data):
     t = symbol('t',
-               'var * {amount: float64, id: int64, name: string, when: datetime}')
+               ('var * {amount: float64, id: int64,'
+                      ' name: string, when: datetime}'))
 
     py_data = into(list, date_data)  # a python version of the collection
 
@@ -354,7 +364,8 @@ def test_datetime_access(date_data):
 
 def test_datetime_access_and_arithmetic(date_data):
     t = symbol('t',
-               'var * {amount: float64, id: int64, name: string, when: datetime}')
+               ('var * {amount: float64, id: int64,'
+                       ' name: string, when: datetime}'))
 
     py_data = into(list, date_data)  # a python version of the collection
 

blaze/compute/tests/test_mysql_compute.py

@@ -50,7 +50,7 @@ def test_agg_sql(db, data):
             nyc.passenger_count as passenger_count
          from
             nyc
-         where nyc.passenger_count < %(passenger_count_1)s) as alias
+         where nyc.passenger_count < %s) as alias
     """
     assert normalize(str(result)) == normalize(expected)
 

blaze/compute/tests/test_pytables_compute.py

@@ -2,21 +2,8 @@
 
 import os
 import pytest
-import pandas as pd
 tb = pytest.importorskip('tables')
 
-
-try:
-    f = pd.HDFStore('foo')
-except (RuntimeError, ImportError) as e:
-    pytest.skip('skipping test_hdfstore.py %s' % e)
-else:
-    f.close()
-    os.remove('foo')
-
-
-from blaze.compatibility import xfail
-
 import numpy as np
 
 from blaze.compute.core import compute
@@ -169,11 +156,17 @@ def test_basic(self, data):
         with pytest.raises(ValueError):
             compute(t.sort('id'), data)
 
-    @xfail(reason='PyTables does not support multiple column sorting')
+    @pytest.mark.xfail(
+        raises=TypeError,
+        reason='PyTables does not support multiple column sorting'
+    )
     def test_multiple_columns(self, data):
         compute(t.sort(['amount', 'id']), data)
 
-    @xfail(reason='PyTables does not support multiple column sorting')
+    @pytest.mark.xfail(
+        raises=TypeError,
+        reason='PyTables does not support multiple column sorting'
+    )
     def test_multiple_columns_sorted_data(self, csi_data):
         compute(t.sort(['amount', 'id']), csi_data)
 
@@ -205,11 +198,17 @@ def test_ascending(self, csi_data):
 class TestIndexSort(object):
     """Fails with a partially sorted index"""
 
-    @xfail(reason='PyTables cannot sort with a standard index')
+    @pytest.mark.xfail(
+        raises=ValueError,
+        reason='PyTables cannot sort with a standard index'
+    )
     def test_basic(self, idx_data):
         compute(t.sort('amount'), idx_data)
 
-    @xfail(reason='PyTables cannot sort with a standard index')
+    @pytest.mark.xfail(
+        raises=ValueError,
+        reason='PyTables cannot sort with a standard index'
+    )
     def test_ascending(self, idx_data):
         compute(t.sort('amount', ascending=False), idx_data)
 
@@ -225,13 +224,15 @@ def pyt():
     fn = 'test.pyt.h5'
     f = tb.open_file(fn, mode='w')
     d = f.create_table('/', 'test', x)
-    yield d
-    d.close()
-    f.close()
     try:
-        os.remove(fn)
-    except OSError:
-        pass
+        yield d
+    finally:
+        d.close()
+        f.close()
+        try:
+            os.remove(fn)
+        except OSError:
+            pass
 
 
 def test_drop(pyt):

blaze/expr/__init__.py

@@ -10,6 +10,5 @@
 from .collections import *
 from .split_apply_combine import *
 from .reductions import *
-from .parser import *
 from .arrays import *
 from . import math

blaze/expr/arithmetic.py

@@ -263,10 +263,9 @@ def scalar_coerce(_, val):
     if val == '':
         raise TypeError('%r is not a valid date' % val)
     dt = dt_parse(val)
-    if dt.time():  # TODO: doesn't work with python 3.5
-        raise TypeError(
-            "Can not coerce %r to type Date, contains time information" % val
-        )
+    if any(x > 0 for x in (dt.hour, dt.minute, dt.second, dt.microsecond)):
+        msg = "Can not coerce %r to type Date, contains time information" 
+        raise TypeError(msg % val)
     return dt.date()