Merged from upstream.

appveyor.yml

@@ -27,12 +27,6 @@ environment:
   IBIS_TEST_MYSQL_PASSWORD: "Password12!"
   IBIS_TEST_MYSQL_DATABASE: "ibis_testing"
 
-  IBIS_TEST_MAPD_HOST: "localhost"
-  IBIS_TEST_MAPD_PORT: "9091"
-  IBIS_TEST_MAPD_USER: "mapd"
-  IBIS_TEST_MAPD_PASSWORD: "mapd"
-  IBIS_TEST_MAPD_DATABASE: "ibis_testing"
-
   IBIS_TEST_SQLITE_DATABASE: "%USERPROFILE%\\ibis_testing.db"
 
   CONDA: "C:\\Miniconda36-x64\\Scripts\\conda"
@@ -55,7 +49,7 @@ test_script:
   - "%CONDA% config --set always_yes true"
   - "%CONDA% create --name \"ibis_%PYTHON_VERSION%\" python=%PYTHON_VERSION% --channel conda-forge"
   - "%ACTIVATE% \"ibis_%PYTHON_VERSION%\""
-  - "%CONDA% install --channel conda-forge pytables numpy sqlalchemy psycopg2 pymapd graphviz click mock plumbum flake8 pytest"
+  - "%CONDA% install --channel conda-forge pytables numpy \"sqlalchemy<1.1.15\" psycopg2 graphviz click mock plumbum flake8 pytest"
   - "%CONDA% list"
   - "pip install -e .\"[sqlite, postgres, mysql, visualization, pandas, csv, hdf5]\""
 

ci/datamgr.py

@@ -71,7 +71,10 @@ def read_tables(names, data_directory):
 def insert_tables(engine, names, data_directory):
     for table, df in read_tables(names, data_directory):
         with engine.begin() as connection:
-            df.to_sql(table, connection, index=False, if_exists='append')
+            df.to_sql(
+                table, connection, index=False, if_exists='append',
+                chunksize=1
+            )
 
 
 @click.group()

ibis/bigquery/compiler.py

@@ -374,6 +374,7 @@ def _formatter(translator, expr):
     ops.ArrayLength: unary('ARRAY_LENGTH'),
 
     ops.Log: _log,
+    ops.Sign: unary('SIGN'),
     ops.Modulus: fixed_arity('MOD', 2),
 
     ops.Date: unary('DATE'),

ibis/expr/datatypes.py

@@ -1,17 +1,3 @@
-# Copyright 2014 Cloudera Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import re
 import six
 import toolz

ibis/expr/operations.py

@@ -493,9 +493,8 @@ class Exp(RealUnaryOp):
 
 
 class Sign(UnaryOp):
-
-    # This is the Impala output for both integers and double/float
-    output_type = rlz.shape_like('arg', dt.float)
+    arg = Arg(rlz.numeric)
+    output_type = rlz.typeof('arg')
 
 
 class Sqrt(RealUnaryOp):

ibis/expr/tests/test_sql_builtins.py

@@ -135,7 +135,7 @@ def test_sign(functional_alltypes, lineitem):
 
     dec_col = lineitem.l_extendedprice
     result = dec_col.sign()
-    assert isinstance(result, ir.FloatingColumn)
+    assert isinstance(result, ir.DecimalColumn)
 
 
 def test_round(functional_alltypes, lineitem):

ibis/impala/compiler.py

@@ -972,6 +972,15 @@ def _string_like(translator, expr):
     )
 
 
+def _sign(translator, expr):
+    arg, = expr.op().args
+    translated_arg = translator.translate(arg)
+    translated_type = _type_to_sql_string(expr.type())
+    if expr.type() != dt.float:
+        return 'CAST(sign({}) AS {})'.format(translated_arg, translated_type)
+    return 'sign({})'.format(translated_arg)
+
+
 _operation_registry = {
     # Unary operations
     ops.NotNull: _not_null,
@@ -995,7 +1004,7 @@ def _string_like(translator, expr):
     ops.Exp: unary('exp'),
     ops.Round: _round,
 
-    ops.Sign: unary('sign'),
+    ops.Sign: _sign,
     ops.Sqrt: unary('sqrt'),
 
     ops.Hash: _hash,

ibis/impala/tests/test_exprs.py

@@ -1,17 +1,3 @@
-# Copyright 2014 Cloudera Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import unittest
 
 import pytest
@@ -370,7 +356,7 @@ def setUp(self):
 
     def test_numeric_unary_builtins(self):
         # No argument functions
-        functions = ['abs', 'ceil', 'floor', 'exp', 'sqrt', 'sign',
+        functions = ['abs', 'ceil', 'floor', 'exp', 'sqrt',
                      ('log', 'ln'),
                      ('approx_median', 'appx_median'),
                      ('approx_nunique', 'ndv'),
@@ -406,6 +392,20 @@ def test_round(self):
         ]
         self._check_expr_cases(cases)
 
+    def test_sign(self):
+        cases = [
+            (
+                self.table.tinyint_col.sign(),
+                'CAST(sign(`tinyint_col`) AS tinyint)'
+            ),
+            (self.table.float_col.sign(), 'sign(`float_col`)'),
+            (
+                self.table.double_col.sign(),
+                'CAST(sign(`double_col`) AS double)'
+            ),
+        ]
+        self._check_expr_cases(cases)
+
     def test_hash(self):
         expr = self.table.int_col.hash()
         assert isinstance(expr, ir.IntegerColumn)

ibis/impala/tests/test_partition.py

@@ -1,17 +1,3 @@
-# Copyright 2014 Cloudera Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import unittest
 
 from posixpath import join as pjoin

ibis/mapd/operations.py

@@ -248,18 +248,6 @@ def _cov(translator, expr):
     )
 
 
-# MATH
-
-def _round(translator, expr):
-    op = expr.op()
-    arg, digits = op.args
-
-    if digits is not None:
-        return _call(translator, 'round', arg, digits)
-    else:
-        return _call(translator, 'round', arg)
-
-
 # STRING
 
 def _length(func_name='length', sql_func_name='CHAR_LENGTH'):
@@ -550,7 +538,6 @@ class ByteLength(ops.StringLength):
     ops.Modulus: fixed_arity('mod', 2),
     ops.Pi: fixed_arity('pi', 0),
     ops.Radians: unary('radians'),
-    ops.Round: _round,
     NumericTruncate: fixed_arity('truncate', 2)
 }
 
@@ -686,6 +673,7 @@ class ByteLength(ops.StringLength):
     ops.Greatest,
     ops.Log2,
     ops.Log,
+    ops.Round,
     # date/time/timestamp
     ops.TimestampFromUNIX,
     ops.Date,

ibis/pandas/client.py

@@ -146,7 +146,9 @@ def infer_pandas_schema(df, schema=None):
             ibis_dtype = dt.dtype(schema[column_name])
         elif pandas_dtype == np.object_:
             inferred_dtype = infer_pandas_dtype(df[column_name].dropna())
-            if inferred_dtype == 'mixed':
+            if inferred_dtype in {'mixed', 'decimal'}:
+                # TODO: in principal we can handle decimal (added in pandas
+                # 0.23)
                 raise TypeError(
                     'Unable to infer type of column {0!r}. Try instantiating '
                     'your table from the client with client.table('

ibis/pandas/core.py

@@ -154,16 +154,30 @@ def execute_with_scope(expr, scope, context=None, **kwargs):
     )
     new_scope = toolz.merge(scope, *pre_executed_scope)
     result = execute_until_in_scope(
-        expr, new_scope, context=context, clients=clients, **kwargs)
+        expr,
+        new_scope,
+        context=context,
+        clients=clients,
+
+        # XXX: we *explicitly* pass in scope and not new_scope here so that
+        # post_execute sees the scope of execute_with_scope, not the scope of
+        # execute_until_in_scope
+        post_execute_=functools.partial(
+            post_execute,
+            scope=scope,
+            context=context,
+            clients=clients,
+            **kwargs
+        ),
+        **kwargs
+    )
 
-    # XXX: we *explicitly* pass in scope and not new_scope here so that
-    # post_execute sees the scope of execute_with_scope, not the scope of
-    # execute_until_in_scope
-    return post_execute(
-        op, result, scope=scope, context=context, clients=clients, **kwargs)
+    return result
 
 
-def execute_until_in_scope(expr, scope, context=None, clients=None, **kwargs):
+def execute_until_in_scope(
+    expr, scope, context=None, clients=None, post_execute_=None, **kwargs
+):
     """Execute until our op is in `scope`.
 
     Parameters
@@ -177,18 +191,23 @@ def execute_until_in_scope(expr, scope, context=None, clients=None, **kwargs):
     # these should never be None
     assert context is not None, 'context is None'
     assert clients is not None, 'clients is None'
+    assert post_execute_ is not None, 'post_execute_ is None'
 
     # base case: our op has been computed (or is a leaf data node), so
     # return the corresponding value
     op = expr.op()
     if op in scope:
         return scope[op]
 
-    new_scope = execute_bottom_up(expr, scope, context=context, **kwargs)
+    new_scope = execute_bottom_up(
+        expr, scope, context=context, post_execute_=post_execute_, **kwargs)
     pre_executor = functools.partial(pre_execute, op, scope=scope, **kwargs)
     new_scope = toolz.merge(new_scope, *map(pre_executor, clients))
     return execute_until_in_scope(
-        expr, new_scope, context=context, clients=clients, **kwargs)
+        expr, new_scope,
+        context=context, clients=clients, post_execute_=post_execute_,
+        **kwargs
+    )
 
 
 def is_computable_arg(op, arg):
@@ -209,7 +228,7 @@ def is_computable_arg(op, arg):
     )
 
 
-def execute_bottom_up(expr, scope, context=None, **kwargs):
+def execute_bottom_up(expr, scope, context=None, post_execute_=None, **kwargs):
     """Execute `expr` bottom-up.
 
     Parameters
@@ -227,6 +246,7 @@ def execute_bottom_up(expr, scope, context=None, **kwargs):
     ]
         A mapping from node to the computed result of that Node
     """
+    assert post_execute_ is not None, 'post_execute_ is None'
     op = expr.op()
 
     # if we're in scope then return the scope, this will then be passed back
@@ -251,7 +271,8 @@ def execute_bottom_up(expr, scope, context=None, **kwargs):
 
     # recursively compute each node's arguments until we've changed type
     scopes = [
-        execute_bottom_up(arg, scope, context=context, **kwargs)
+        execute_bottom_up(
+            arg, scope, context=context, post_execute_=post_execute_, **kwargs)
         if hasattr(arg, 'op') else {arg: arg}
         for arg in computable_args
     ]
@@ -273,7 +294,8 @@ def execute_bottom_up(expr, scope, context=None, **kwargs):
         for arg in computable_args
     ]
     result = execute_node(op, *data, scope=scope, context=context, **kwargs)
-    return {op: result}
+    computed = post_execute_(op, result)
+    return {op: computed}
 
 
 def execute(expr, params=None, scope=None, context=None, **kwargs):

ibis/pandas/execution/constants.py

@@ -71,4 +71,5 @@
     ops.FloorDivide: operator.floordiv,
     ops.Modulus: operator.mod,
     ops.Power: operator.pow,
+    ops.IdenticalTo: lambda x, y: (x == y) | (pd.isnull(x) & pd.isnull(y))
 }