278 changes: 230 additions & 48 deletions ibis/client.py

Large diffs are not rendered by default.

65 changes: 0 additions & 65 deletions ibis/common.py

This file was deleted.

File renamed without changes.
100 changes: 100 additions & 0 deletions ibis/common/exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
"""Module for exceptions."""
# Copyright 2014 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Callable


class IbisError(Exception):
"""IbisError."""


class InternalError(IbisError):
"""InternalError."""


class IntegrityError(IbisError):
"""IntegrityError."""


class ExpressionError(IbisError):
"""ExpressionError."""


class RelationError(ExpressionError):
"""RelationError."""


class TranslationError(IbisError):
"""TranslationError."""


class OperationNotDefinedError(TranslationError):
"""OperationNotDefinedError."""


class UnsupportedOperationError(TranslationError):
"""UnsupportedOperationError."""


class UnsupportedBackendType(TranslationError):
"""UnsupportedBackendType."""


class UnboundExpressionError(ValueError, IbisError):
"""UnboundExpressionError."""


class IbisInputError(ValueError, IbisError):
"""IbisInputError."""


class IbisTypeError(TypeError, IbisError):
"""IbisTypeError."""


class InputTypeError(IbisTypeError):
"""InputTypeError."""


class UnsupportedArgumentError(IbisError):
"""UnsupportedArgumentError."""


def mark_as_unsupported(f: Callable) -> Callable:
"""Decorate an unsupported method.
Parameters
----------
f : callable
Returns
-------
callable
Raises
------
UnsupportedOperationError
"""
# function that raises UnsupportedOperationError
def _mark_as_unsupported(self):
raise UnsupportedOperationError(
'Method `{}` are unsupported by class `{}`.'.format(
f.__name__, self.__class__.__name__
)
)

_mark_as_unsupported.__doc__ = f.__doc__
_mark_as_unsupported.__name__ = f.__name__

return _mark_as_unsupported
162 changes: 162 additions & 0 deletions ibis/common/geospatial.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
from typing import Iterable, TypeVar

import ibis.expr.types as ir
from ibis.common import exceptions as ex

IS_SHAPELY_AVAILABLE = False
try:
import shapely

IS_SHAPELY_AVAILABLE = True
except ImportError:
...

NumberType = TypeVar('NumberType', int, float)
# Geometry primitives (2D)
PointType = Iterable[NumberType]
LineStringType = Iterable[PointType]
PolygonType = Iterable[LineStringType]
# Multipart geometries (2D)
MultiPointType = Iterable[PointType]
MultiLineStringType = Iterable[LineStringType]
MultiPolygonType = Iterable[PolygonType]


def _format_point_value(value: PointType) -> str:
"""Convert a iterable with a point to text."""
return ' '.join(str(v) for v in value)


def _format_linestring_value(value: LineStringType, nested=False) -> str:
"""Convert a iterable with a linestring to text."""
template = '({})' if nested else '{}'
if not isinstance(value[0], (tuple, list)):
msg = '{} structure expected: LineStringType'.format(
'Data' if not nested else 'Inner data'
)
raise ex.IbisInputError(msg)
return template.format(
', '.join(_format_point_value(point) for point in value)
)


def _format_polygon_value(value: PolygonType, nested=False) -> str:
"""Convert a iterable with a polygon to text."""
template = '({})' if nested else '{}'
if not isinstance(value[0][0], (tuple, list)):
msg = '{} data structure expected: PolygonType'.format(
'Data' if not nested else 'Inner data'
)
raise ex.IbisInputError(msg)

return template.format(
', '.join(
_format_linestring_value(line, nested=True) for line in value
)
)


def _format_multipoint_value(value: MultiPointType) -> str:
"""Convert a iterable with a multipoint to text."""
if not isinstance(value[0], (tuple, list)):
raise ex.IbisInputError('Data structure expected: MultiPointType')
return ', '.join(
'({})'.format(_format_point_value(point)) for point in value
)


def _format_multilinestring_value(value: MultiLineStringType) -> str:
"""Convert a iterable with a multilinestring to text."""
if not isinstance(value[0][0], (tuple, list)):
raise ex.IbisInputError('Data structure expected: MultiLineStringType')
return ', '.join(
'({})'.format(_format_linestring_value(line)) for line in value
)


def _format_multipolygon_value(value: MultiPolygonType) -> str:
"""Convert a iterable with a multipolygon to text."""
if not isinstance(value[0][0], (tuple, list)):
raise ex.IbisInputError('Data structure expected: MultiPolygonType')
return ', '.join(
_format_polygon_value(polygon, nested=True) for polygon in value
)


def _format_geo_metadata(op, value: str, inline_metadata: bool = False) -> str:
"""Format a geometry/geography text when it is necessary."""
srid = op.args[1].srid
geotype = op.args[1].geotype

if inline_metadata:
value = "'{}{}'{}".format(
'SRID={};'.format(srid) if srid else '',
value,
'::{}'.format(geotype) if geotype else '',
)
return value

geofunc = (
'ST_GeogFromText' if geotype == 'geography' else 'ST_GeomFromText'
)

value = repr(value)
if srid:
value += ', {}'.format(srid)

return "{}({})".format(geofunc, value)


def translate_point(value: Iterable) -> str:
"""Translate a point to WKT."""
return "POINT ({})".format(_format_point_value(value))


def translate_linestring(value: Iterable) -> str:
"""Translate a linestring to WKT."""
return "LINESTRING ({})".format(_format_linestring_value(value))


def translate_polygon(value: Iterable) -> str:
"""Translate a polygon to WKT."""
return "POLYGON ({})".format(_format_polygon_value(value))


def translate_multilinestring(value: Iterable) -> str:
"""Translate a multilinestring to WKT."""
return "MULTILINESTRING ({})".format(_format_multilinestring_value(value))


def translate_multipoint(value: Iterable) -> str:
"""Translate a multipoint to WKT."""
return "MULTIPOINT ({})".format(_format_multipoint_value(value))


def translate_multipolygon(value: Iterable) -> str:
"""Translate a multipolygon to WKT."""
return "MULTIPOLYGON ({})".format(_format_multipolygon_value(value))


def translate_literal(expr, inline_metadata: bool = False) -> str:
op = expr.op()
value = op.value

if IS_SHAPELY_AVAILABLE and isinstance(
value, shapely.geometry.base.BaseGeometry
):
result = value.wkt
elif isinstance(expr, ir.PointScalar):
result = translate_point(value)
elif isinstance(expr, ir.LineStringScalar):
result = translate_linestring(value)
elif isinstance(expr, ir.PolygonScalar):
result = translate_polygon(value)
elif isinstance(expr, ir.MultiLineStringScalar):
result = translate_multilinestring(value)
elif isinstance(expr, ir.MultiPointScalar):
result = translate_multipoint(value)
elif isinstance(expr, ir.MultiPolygonScalar):
result = translate_multipolygon(value)
else:
raise ex.UnboundExpressionError('Geo Spatial type not supported.')
return _format_geo_metadata(op, result, inline_metadata)
4 changes: 4 additions & 0 deletions ibis/compat.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
"""Module for compatible functions."""
import operator
import sys

import toolz

Expand All @@ -23,3 +25,5 @@


to_date = toolz.compose(operator.methodcaller('date'), to_datetime)

PY38 = sys.version_info >= (3, 8, 0)
331 changes: 242 additions & 89 deletions ibis/config.py

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions ibis/config_init.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
"""Initialize default configuration."""
# Copyright 2014 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
Expand Down
8 changes: 6 additions & 2 deletions ibis/expr/analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@
import ibis.expr.operations as ops
import ibis.expr.types as ir
from ibis import util
from ibis.common import ExpressionError, IbisTypeError, RelationError
from ibis.common.exceptions import (
ExpressionError,
IbisTypeError,
RelationError,
)
from ibis.expr.schema import HasSchema
from ibis.expr.window import window

Expand Down Expand Up @@ -165,7 +169,7 @@ def reduction_to_aggregation(expr, default_name='tmp'):
named_expr = expr.name(default_name)

if len(tables) == 1:
table, = tables
(table,) = tables
return table.aggregate([named_expr]), name
else:
return ScalarAggregate(expr, None, default_name).get_result()
Expand Down
232 changes: 213 additions & 19 deletions ibis/expr/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import toolz

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.analysis as _L
import ibis.expr.analytics as _analytics
import ibis.expr.datatypes as dt
Expand Down Expand Up @@ -59,6 +59,12 @@
MapColumn,
MapScalar,
MapValue,
MultiLineStringColumn,
MultiLineStringScalar,
MultiLineStringValue,
MultiPointColumn,
MultiPointScalar,
MultiPointValue,
MultiPolygonColumn,
MultiPolygonScalar,
MultiPolygonValue,
Expand Down Expand Up @@ -134,8 +140,15 @@
'geo_d_within',
'geo_envelope',
'geo_equals',
'geo_geometry_n',
'geo_geometry_type',
'geo_intersection',
'geo_intersects',
'geo_is_valid',
'geo_line_locate_point',
'geo_line_merge',
'geo_line_substring',
'geo_ordering_equals',
'geo_overlaps',
'geo_touches',
'geo_distance',
Expand All @@ -150,13 +163,15 @@
'geo_srid',
'geo_start_point',
'geo_transform',
'geo_unary_union',
'geo_union',
'geo_within',
'geo_x',
'geo_x_max',
'geo_x_min',
'geo_y',
'geo_y_max',
'geo_y_min',
'geo_within',
'greatest',
'ifelse',
'infer_dtype',
Expand Down Expand Up @@ -1602,20 +1617,19 @@ def covariance(left, right, where=None, how='sample'):
# GeoSpatial API


def geo_area(arg, use_spheroid=None):
def geo_area(arg):
"""
Compute area of a geo spatial data
Parameters
----------
arg : geometry or geography
use_spheroid: default None
Returns
-------
area : double scalar
"""
op = ops.GeoArea(arg, use_spheroid)
op = ops.GeoArea(arg)
return op.to_expr()


Expand Down Expand Up @@ -1841,6 +1855,39 @@ def geo_equals(left, right):
return op.to_expr()


def geo_geometry_n(arg, n):
"""
Get the 1-based Nth geometry of a multi geometry.
Parameters
----------
arg : geometry
n : integer
Returns
-------
geom : geometry scalar
"""
op = ops.GeoGeometryN(arg, n)
return op.to_expr()


def geo_geometry_type(arg):
"""
Get the type of a geometry.
Parameters
----------
arg : geometry
Returns
-------
type : string scalar
"""
op = ops.GeoGeometryType(arg)
return op.to_expr()


def geo_intersects(left, right):
"""
Check if the geometries share any points.
Expand All @@ -1858,6 +1905,107 @@ def geo_intersects(left, right):
return op.to_expr()


def geo_is_valid(arg):
"""
Check if the geometry is valid.
Parameters
----------
arg : geometry
Returns
-------
valid : bool scalar
"""
op = ops.GeoIsValid(arg)
return op.to_expr()


def geo_line_locate_point(left, right):
"""
Locate the distance a point falls along the length of a line.
Returns a float between zero and one representing the location of the
closest point on the linestring to the given point, as a fraction of the
total 2d line length.
Parameters
----------
left : linestring
right: point
Returns
-------
distance: float scalar
"""
op = ops.GeoLineLocatePoint(left, right)
return op.to_expr()


def geo_line_merge(arg):
"""
Merge a MultiLineString into a LineString.
Returns a (set of) LineString(s) formed by sewing together the
constituent line work of a MultiLineString. If a geometry other than
a LineString or MultiLineString is given, this will return an empty
geometry collection.
Parameters
----------
arg : (multi)linestring
Returns
-------
merged: geometry scalar
"""
op = ops.GeoLineMerge(arg)
return op.to_expr()


def geo_line_substring(arg, start, end):
"""
Clip a substring from a LineString.
Returns a linestring that is a substring of the input one, starting
and ending at the given fractions of the total 2d length. The second
and third arguments are floating point values between zero and one.
This only works with linestrings.
Parameters
----------
arg: linestring
start: float
end: float
Returns
-------
substring: linestring scalar
"""
op = ops.GeoLineSubstring(arg, start, end)
return op.to_expr()


def geo_ordering_equals(left, right):
"""
Check if two geometries are equal and have the same point ordering.
Returns true if the two geometries are equal and the coordinates
are in the same order.
Parameters
----------
left : geometry
right : geometry
Returns
-------
ordering_equals : bool scalar
"""
op = ops.GeoOrderingEquals(left, right)
return op.to_expr()


def geo_overlaps(left, right):
"""
Check if the geometries share space, are of the same dimension,
Expand Down Expand Up @@ -1894,55 +2042,52 @@ def geo_touches(left, right):
return op.to_expr()


def geo_distance(left, right, use_spheroid=None):
def geo_distance(left, right):
"""
Compute distance between two geo spatial data
Parameters
----------
left : geometry or geography
right : geometry or geography
use_spheroid : default None
Returns
-------
distance : double scalar
"""
op = ops.GeoDistance(left, right, use_spheroid)
op = ops.GeoDistance(left, right)
return op.to_expr()


def geo_length(arg, use_spheroid=None):
def geo_length(arg):
"""
Compute length of a geo spatial data
Parameters
----------
arg : geometry or geography
use_spheroid : default None
Returns
-------
length : double scalar
"""
op = ops.GeoLength(arg, use_spheroid)
op = ops.GeoLength(arg)
return op.to_expr()


def geo_perimeter(arg, use_spheroid=None):
def geo_perimeter(arg):
"""
Compute perimeter of a geo spatial data
Parameters
----------
arg : geometry or geography
use_spheroid : default None
Returns
-------
perimeter : double scalar
"""
op = ops.GeoPerimeter(arg, use_spheroid)
op = ops.GeoPerimeter(arg)
return op.to_expr()


Expand All @@ -1965,6 +2110,47 @@ def geo_max_distance(left, right):
return op.to_expr()


def geo_unary_union(arg):
"""
Aggregate a set of geometries into a union.
This corresponds to the aggregate version of the PostGIS ST_Union.
We give it a different name (following the corresponding method
in GeoPandas) to avoid name conflicts with the non-aggregate version.
Parameters
----------
arg : geometry column
Returns
-------
union : geometry scalar
"""
expr = ops.GeoUnaryUnion(arg).to_expr()
expr = expr.name('union')
return expr


def geo_union(left, right):
"""
Merge two geometries into a union geometry.
Returns the pointwise union of the two geometries.
This corresponds to the non-aggregate version the PostGIS ST_Union.
Parameters
----------
left : geometry
right : geometry
Returns
-------
union : geometry scalar
"""
op = ops.GeoUnion(left, right)
return op.to_expr()


def geo_x(arg):
"""Return the X coordinate of the point, or NULL if not available.
Input must be a point
Expand Down Expand Up @@ -2344,12 +2530,19 @@ def geo_transform(arg, srid):
end_point=geo_end_point,
envelope=geo_envelope,
equals=geo_equals,
geometry_n=geo_geometry_n,
geometry_type=geo_geometry_type,
intersection=geo_intersection,
intersects=geo_intersects,
is_valid=geo_is_valid,
line_locate_point=geo_line_locate_point,
line_merge=geo_line_merge,
line_substring=geo_line_substring,
length=geo_length,
max_distance=geo_max_distance,
n_points=geo_n_points,
n_rings=geo_n_rings,
ordering_equals=geo_ordering_equals,
overlaps=geo_overlaps,
perimeter=geo_perimeter,
point_n=geo_point_n,
Expand All @@ -2359,6 +2552,7 @@ def geo_transform(arg, srid):
start_point=geo_start_point,
touches=geo_touches,
transform=geo_transform,
union=geo_union,
within=geo_within,
x=geo_x,
x_max=geo_x_max,
Expand All @@ -2367,7 +2561,7 @@ def geo_transform(arg, srid):
y_max=geo_y_max,
y_min=geo_y_min,
)
_geospatial_column_methods = dict()
_geospatial_column_methods = dict(unary_union=geo_unary_union)

_add_methods(ir.GeoSpatialValue, _geospatial_value_methods)
_add_methods(ir.GeoSpatialColumn, _geospatial_column_methods)
Expand Down Expand Up @@ -3295,15 +3489,15 @@ def between_time(arg, lower, upper, timezone=None):
-------
BooleanValue
"""

if isinstance(arg.op(), ops.Time):
op = arg.op()
if isinstance(op, ops.Time):
# Here we pull out the first argument to the underlying Time operation
# which is by definition (in _timestamp_value_methods) a
# TimestampValue. We do this so that we can potentially specialize the
# "between time" operation for timestamp_value_expr.time().between().
# A similar mechanism is triggered when creating expressions like
# t.column.distinct().count(), which is turned into t.column.nunique().
arg = arg.op().args[0]
arg = op.arg
if timezone is not None:
arg = arg.cast(dt.Timestamp(timezone=timezone))
op = ops.BetweenTime(arg, lower, upper)
Expand Down Expand Up @@ -4087,6 +4281,6 @@ def prevent_rewrite(expr, client=None):
sql_query_result : ir.TableExpr
"""
if client is None:
client, = ibis.client.find_backends(expr)
(client,) = ibis.client.find_backends(expr)
query = client.compile(expr)
return ops.SQLQueryResult(query, expr.schema(), client).to_expr()
293 changes: 277 additions & 16 deletions ibis/expr/datatypes.py

Large diffs are not rendered by default.

148 changes: 144 additions & 4 deletions ibis/expr/operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
import itertools
import operator
from contextlib import suppress
from typing import List

import toolz

import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.expr.rules as rlz
import ibis.expr.schema as sch
Expand Down Expand Up @@ -1026,8 +1027,10 @@ def __init__(self, expr, window):
window = window.bind(table)

if window.max_lookback is not None:
error_msg = ("'max lookback' windows must be ordered "
"by a timestamp column")
error_msg = (
"'max lookback' windows must be ordered "
"by a timestamp column"
)
if len(window._order_by) != 1:
raise com.IbisInputError(error_msg)
order_var = window._order_by[0].op().args[0]
Expand Down Expand Up @@ -1730,6 +1733,13 @@ def __init__(self, left, right, predicates, by, tolerance):
super().__init__(left, right, predicates)
self.by = _clean_join_predicates(self.left, self.right, by)
self.tolerance = tolerance
self._validate_args(['by', 'tolerance'])

def _validate_args(self, args: List[str]):
for arg in args:
argument = self.signature[arg]
value = argument.validate(getattr(self, arg))
setattr(self, arg, value)


class Union(TableNode, HasSchema):
Expand Down Expand Up @@ -2943,6 +2953,10 @@ def equals(self, other, cache=None):
and self.dtype.equals(other.dtype, cache=cache)
)

@property
def inputs(self):
return ()

def root_tables(self):
return []

Expand Down Expand Up @@ -3052,6 +3066,20 @@ class GeoEquals(GeoSpatialBinOp):
output_type = rlz.shape_like('args', dt.boolean)


class GeoGeometryN(GeoSpatialUnOp):
"""Returns the Nth Geometry of a Multi geometry."""

n = Arg(rlz.integer)

output_type = rlz.shape_like('args', dt.geometry)


class GeoGeometryType(GeoSpatialUnOp):
"""Returns the type of the geometry."""

output_type = rlz.shape_like('args', dt.string)


class GeoIntersects(GeoSpatialBinOp):
"""Returns True if the Geometries/Geography “spatially intersect in 2D”
- (share any portion of space) and False if they don’t (they are Disjoint).
Expand All @@ -3060,6 +3088,69 @@ class GeoIntersects(GeoSpatialBinOp):
output_type = rlz.shape_like('args', dt.boolean)


class GeoIsValid(GeoSpatialUnOp):
"""Returns true if the geometry is well-formed."""

output_type = rlz.shape_like('args', dt.boolean)


class GeoLineLocatePoint(GeoSpatialBinOp):
"""
Locate the distance a point falls along the length of a line.
Returns a float between zero and one representing the location of the
closest point on the linestring to the given point, as a fraction of the
total 2d line length.
"""

left = Arg(rlz.linestring)
right = Arg(rlz.point)

output_type = rlz.shape_like('args', dt.halffloat)


class GeoLineMerge(GeoSpatialUnOp):
"""
Merge a MultiLineString into a LineString.
Returns a (set of) LineString(s) formed by sewing together the
constituent line work of a multilinestring. If a geometry other than
a linestring or multilinestring is given, this will return an empty
geometry collection.
"""

output_type = rlz.shape_like('args', dt.geometry)


class GeoLineSubstring(GeoSpatialUnOp):
"""
Clip a substring from a LineString.
Returns a linestring that is a substring of the input one, starting
and ending at the given fractions of the total 2d length. The second
and third arguments are floating point values between zero and one.
This only works with linestrings.
"""

arg = Arg(rlz.linestring)

start = Arg(rlz.floating)
end = Arg(rlz.floating)

output_type = rlz.shape_like('args', dt.linestring)


class GeoOrderingEquals(GeoSpatialBinOp):
"""
Check if two geometries are equal and have the same point ordering.
Returns true if the two geometries are equal and the coordinates
are in the same order.
"""

output_type = rlz.shape_like('args', dt.boolean)


class GeoOverlaps(GeoSpatialBinOp):
"""Returns True if the Geometries share space, are of the same dimension,
but are not completely contained by each other."""
Expand All @@ -3074,6 +3165,21 @@ class GeoTouches(GeoSpatialBinOp):
output_type = rlz.shape_like('args', dt.boolean)


class GeoUnaryUnion(Reduction):
"""Returns the pointwise union of the geometries in the column."""

arg = Arg(rlz.column(rlz.geospatial))

def output_type(self):
return dt.geometry.scalar_type()


class GeoUnion(GeoSpatialBinOp):
"""Returns the pointwise union of the two geometries."""

output_type = rlz.shape_like('args', dt.geometry)


class GeoArea(GeoSpatialUnOp):
"""Area of the geo spatial data"""

Expand Down Expand Up @@ -3158,7 +3264,6 @@ class GeoEndPoint(GeoSpatialUnOp):
output_type = rlz.shape_like('arg', dt.point)


# TODO: https://postgis.net/docs/ST_PointN.html
class GeoPointN(GeoSpatialUnOp):
"""Return the Nth point in a single linestring in the geometry.
Negative values are counted backwards from the end of the LineString,
Expand Down Expand Up @@ -3192,6 +3297,7 @@ class GeoSRID(GeoSpatialUnOp):

class GeoSetSRID(GeoSpatialUnOp):
"""Set the spatial reference identifier for the ST_Geometry."""

srid = Arg(rlz.integer)
output_type = rlz.shape_like('args', dt.geometry)

Expand All @@ -3217,6 +3323,7 @@ class GeoDFullyWithin(GeoSpatialBinOp):
"""Returns True if the geometries are fully within the specified distance
of one another.
"""

distance = Arg(rlz.floating)

output_type = rlz.shape_like('args', dt.boolean)
Expand All @@ -3226,6 +3333,7 @@ class GeoDWithin(GeoSpatialBinOp):
"""Returns True if the geometries are within the specified distance
of one another.
"""

distance = Arg(rlz.floating)

output_type = rlz.shape_like('args', dt.boolean)
Expand Down Expand Up @@ -3319,3 +3427,35 @@ class GeoAsText(GeoSpatialUnOp):
"""

output_type = rlz.shape_like('arg', dt.string)


class ElementWiseVectorizedUDF(ValueOp):
"""Node for element wise UDF.
"""

func = Arg(callable)
func_args = Arg(list)
input_type = Arg(rlz.shape_like('func_args'))
_output_type = Arg(rlz.noop)

def __init__(self, func, args, input_type, output_type):
self.func = func
self.func_args = args
self.input_type = input_type
self._output_type = output_type

@property
def inputs(self):
return self.func_args

def output_type(self):
return self._output_type.column_type()

def root_tables(self):
result = list(
toolz.unique(
toolz.concat(arg._root_tables() for arg in self.func_args)
)
)

return result
4 changes: 3 additions & 1 deletion ibis/expr/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from contextlib import suppress
from itertools import product, starmap

import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.expr.schema as sch
import ibis.expr.types as ir
Expand Down Expand Up @@ -271,6 +271,8 @@ def array_of(inner, arg):
point = value(dt.Point)
linestring = value(dt.LineString)
polygon = value(dt.Polygon)
multilinestring = value(dt.MultiLineString)
multipoint = value(dt.MultiPoint)
multipolygon = value(dt.MultiPolygon)


Expand Down
9 changes: 7 additions & 2 deletions ibis/expr/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from multipledispatch import Dispatcher

import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.util as util

Expand Down Expand Up @@ -33,7 +33,12 @@ def __init__(self, names, types):
self._name_locs = dict((v, i) for i, v in enumerate(self.names))

if len(self._name_locs) < len(self.names):
raise com.IntegrityError('Duplicate column names')
duplicate_names = list(self.names)
for v in self._name_locs.keys():
duplicate_names.remove(v)
raise com.IntegrityError(
'Duplicate column name(s): {}'.format(duplicate_names)
)

def __repr__(self):
space = 2 + max(map(len, self.names), default=0)
Expand Down
37 changes: 25 additions & 12 deletions ibis/expr/signature.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import inspect
from collections import OrderedDict

import ibis.expr.rules as rlz
Expand Down Expand Up @@ -101,19 +102,31 @@ def from_dtypes(cls, dtypes):
)

def validate(self, *args, **kwargs):
result = []
for i, (name, argument) in enumerate(self.items()):
if i < len(args):
if name in kwargs:
raise TypeError(
'Got multiple values for argument {}'.format(name)
)
value = argument.validate(args[i], name=name)
elif name in kwargs:
value = argument.validate(kwargs[name], name=name)
else:
value = argument.validate(name=name)
parameters = [
inspect.Parameter(
name,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
default=_undefined,
)
for (name, argument) in self.items()
]
sig = inspect.Signature(parameters)
bindings = sig.bind(*args, **kwargs)

# The inspect.Parameter objects in parameters all have default
# value _undefined, which will be bound to all arguments that weren't
# passed in.
bindings.apply_defaults()

result = []
for (name, arg_value) in bindings.arguments.items():
argument = self[name]
# If this arg wasn't passed in: since argument.default has the
# correct value and _undefined was given as the default for the
# Parameter object corresponding to this argument, arg_value got
# the value _undefined when bindings.apply_defaults() was called,
# so the behavior of argument.validate here is correct.
value = argument.validate(arg_value, name=name)
result.append((name, value))

return result
Expand Down
30 changes: 21 additions & 9 deletions ibis/expr/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
import pytest

import ibis
from ibis.expr.tests.mocks import GeoMockConnection, MockConnection
import ibis.common.exceptions as com
from ibis.expr.tests.mocks import MockConnection


@pytest.fixture
Expand Down Expand Up @@ -97,11 +98,22 @@ def lineitem(con):
return con.table('tpch_lineitem')


@pytest.fixture
def geo_con():
return GeoMockConnection()


@pytest.fixture
def geo_table(geo_con):
return geo_con.table('geo')
@pytest.hookimpl(hookwrapper=True)
def pytest_pyfunc_call(pyfuncitem):
"""Dynamically add an xfail marker for specific backends."""
outcome = yield
try:
outcome.get_result()
except (
com.OperationNotDefinedError,
com.UnsupportedOperationError,
com.UnsupportedBackendType,
NotImplementedError,
) as e:
markers = list(pyfuncitem.iter_markers(name="xfail_unsupported"))
assert (
len(markers) == 1
), "More than one xfail_unsupported marker found on test {}".format(
pyfuncitem
)
pytest.xfail(reason=repr(e))
64 changes: 37 additions & 27 deletions ibis/expr/tests/mocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@
# limitations under the License.

import abc

import pytest

from ibis.client import SQLClient
from ibis.expr.schema import Schema
import ibis.expr.types as ir
import ibis.sql.alchemy as alch # noqa: E402
from ibis.client import SQLClient
from ibis.expr.schema import Schema


class BaseMockConnection(SQLClient, metaclass=abc.ABCMeta):
Expand Down Expand Up @@ -419,42 +420,51 @@ def _build_ast(self, expr, context):
return build_ast(expr, context)


class GeoMockConnection(SQLClient):
GEO_TABLE = {
'geo': [
('id', 'int32'),
('geo_point', 'point'),
('geo_linestring', 'linestring'),
('geo_polygon', 'polygon'),
('geo_multipolygon', 'multipolygon'),
]
}


class GeoMockConnectionPostGIS(MockAlchemyConnection):
_tables = GEO_TABLE

def __init__(self):
super().__init__()
self.executed_queries = []

def _get_table_schema(self, name):
return Schema.from_tuples(self._tables[name])

@property
def dialect(self):
from ibis.mapd.compiler import MapDDialect
from ibis.sql.postgres.compiler import PostgreSQLDialect

return MapDDialect
return PostgreSQLDialect

_tables = {
'geo': [
('id', 'int32'),
('geo_point', 'point'),
('geo_linestring', 'linestring'),
('geo_polygon', 'polygon'),
('geo_multipolygon', 'multipolygon'),
]
}

class GeoMockConnectionOmniSciDB(SQLClient):
_tables = GEO_TABLE

def __init__(self):
super().__init__()
self.executed_queries = []

def _get_table_schema(self, name):
# name = name.replace('`', '')
return Schema.from_tuples(self._tables[name])

def _build_ast(self, expr, context):
from ibis.mapd.compiler import build_ast
@property
def dialect(self):
from ibis.omniscidb.compiler import OmniSciDBDialect

return build_ast(expr, context)
return OmniSciDBDialect

def execute(self, expr, limit=None, params=None):
ast = self._build_ast_ensure_limit(expr, limit, params=params)
for query in ast.queries:
self.executed_queries.append(query.compile())
return None
def _build_ast(self, expr, context):
from ibis.omniscidb.compiler import build_ast

def compile(self, expr, limit=None, params=None):
ast = self._build_ast_ensure_limit(expr, limit, params=params)
queries = [q.compile() for q in ast.queries]
return queries[0] if len(queries) == 1 else queries
return build_ast(expr, context)
2 changes: 1 addition & 1 deletion ibis/expr/tests/test_analysis.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import pytest

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.analysis as L
import ibis.expr.operations as ops
from ibis.tests.util import assert_equal
Expand Down
75 changes: 73 additions & 2 deletions ibis/expr/tests/test_datatypes.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import datetime
from collections import OrderedDict

import pandas as pd
import pytest
import pytz
from multipledispatch.conflict import ambiguities
from pytest import param

import ibis
import ibis.expr.datatypes as dt
from ibis.common import IbisTypeError
from ibis.common.exceptions import IbisTypeError


def test_validate_type():
Expand Down Expand Up @@ -40,6 +42,14 @@ def test_validate_type():
('polygon;4326', dt.polygon),
('polygon;4326:geometry', dt.polygon),
('polygon;4326:geography', dt.polygon),
('multilinestring', dt.multilinestring),
('multilinestring;4326', dt.multilinestring),
('multilinestring;4326:geometry', dt.multilinestring),
('multilinestring;4326:geography', dt.multilinestring),
('multipoint', dt.multipoint),
('multipoint;4326', dt.multipoint),
('multipoint;4326:geometry', dt.multipoint),
('multipoint;4326:geography', dt.multipoint),
('multipolygon', dt.multipolygon),
('multipolygon;4326', dt.multipolygon),
('multipolygon;4326:geometry', dt.multipolygon),
Expand Down Expand Up @@ -201,6 +211,8 @@ def test_char_varchar_invalid(spec):
('point', dt.point),
('linestring', dt.linestring),
('polygon', dt.polygon),
('multilinestring', dt.multilinestring),
('multipoint', dt.multipoint),
('multipolygon', dt.multipolygon),
],
)
Expand Down Expand Up @@ -357,7 +369,13 @@ def test_time_valid():
('foo', dt.string),
(datetime.date.today(), dt.date),
(datetime.datetime.now(), dt.timestamp),
(datetime.timedelta(days=3), dt.interval),
(datetime.timedelta(days=3), dt.Interval(unit='D')),
(pd.Timedelta('5 hours'), dt.Interval(unit='h')),
(pd.Timedelta('7 minutes'), dt.Interval(unit='m')),
(datetime.timedelta(seconds=9), dt.Interval(unit='s')),
(pd.Timedelta('11 milliseconds'), dt.Interval(unit='ms')),
(datetime.timedelta(microseconds=15), dt.Interval(unit='us')),
(pd.Timedelta('17 nanoseconds'), dt.Interval(unit='ns')),
# numeric types
(5, dt.int8),
(5, dt.int8),
Expand Down Expand Up @@ -407,6 +425,51 @@ def test_time_valid():
]
),
),
param(
datetime.timedelta(hours=5),
dt.Interval(unit='h'),
id='dateime hours',
marks=pytest.mark.xfail(
reason='Hour conversion from datetime.timedelta to ibis '
'interval not supported'
),
),
param(
datetime.timedelta(minutes=7),
dt.Interval(unit='m'),
id='dateime minutes',
marks=pytest.mark.xfail(
reason='Minute conversion from datetime.timedelta to ibis '
'interval not supported'
),
),
param(
datetime.timedelta(milliseconds=11),
dt.Interval(unit='ms'),
id='dateime milliseconds',
marks=pytest.mark.xfail(
reason='Millisecond conversion from datetime.timedelta to '
'ibis interval not supported'
),
),
param(
pd.Timedelta('3', unit='W'),
dt.Interval(unit='W'),
id='weeks',
marks=pytest.mark.xfail(
reason='Week conversion from Timedelta to ibis interval '
'not supported'
),
),
param(
None,
dt.Interval(unit='Y'),
id='years',
marks=pytest.mark.xfail(
reason='Year conversion from Timedelta to ibis interval '
'not supported'
),
),
],
)
def test_infer_dtype(value, expected_dtype):
Expand Down Expand Up @@ -471,3 +534,11 @@ def test_implicitly_uncastable_values(source, target, value):

def test_no_infer_ambiguities():
assert not ambiguities(dt.infer.funcs)


def test_struct_datatype_subclass_from_tuples():
class MyStruct(dt.Struct):
pass

dtype = MyStruct.from_tuples([('a', 'int64')])
assert isinstance(dtype, MyStruct)
215 changes: 162 additions & 53 deletions ibis/expr/tests/test_geospatial.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,66 +2,175 @@
import pytest

import ibis
from ibis.expr.tests.mocks import (
GeoMockConnectionOmniSciDB,
GeoMockConnectionPostGIS,
)

pytest.importorskip('geoalchemy2')
pytest.importorskip('shapely')
pytest.importorskip('geopandas')

@pytest.mark.parametrize(
'modifier', ['', ';4326', ';4326:geometry', ';4326:geography']
)
def test_geo_literals_smoke(modifier):
"""Smoke tests for geo spatial literals"""
point = (0, 1)
ibis.literal(point, type='point{}'.format(modifier))
pytest.mark.postgis
pytest.mark.omniscidb

linestring = [(0, 1), (2, 3)]
ibis.literal(linestring, type='linestring{}'.format(modifier))
mock_omniscidb = GeoMockConnectionOmniSciDB()
mock_postgis = GeoMockConnectionPostGIS()

polygon = [
((0, 0), (4, 0), (4, 4), (0, 4), (0, 0)),
((1, 1), (2, 1), (2, 2), (1, 2), (1, 1)),
]
ibis.literal(polygon, type='polygon{}'.format(modifier))

polygon1 = (
((0, 0), (4, 0), (4, 4), (0, 4), (0, 0)),
((1, 1), (2, 1), (2, 2), (1, 2), (1, 1)),
@pytest.mark.parametrize('backend', [mock_postgis])
@pytest.mark.parametrize(
'modifier',
[
{},
{'srid': '4326'},
{'srid': '4326', 'geo_type': 'geometry'},
{'srid': '4326', 'geo_type': 'geography'},
],
)
@pytest.mark.parametrize(
'shape,value,expected',
[
# Geometry primitives (2D)
('point', (30, 10), 'POINT (30 10)'),
(
'linestring',
((30, 10), (10, 30), (40, 40)),
'LINESTRING (30 10, 10 30, 40 40)',
),
(
'polygon',
(
((35, 10), (45, 45), (15, 40), (10, 20), (35, 10)),
((20, 30), (35, 35), (30, 20), (20, 30)),
),
(
'POLYGON ((35 10, 45 45, 15 40, 10 20, 35 10), '
+ '(20 30, 35 35, 30 20, 20 30))'
),
),
(
'polygon',
(((30, 10), (40, 40), (20, 40), (10, 20), (30, 10)),),
'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))',
),
# Multipart geometries (2D)
(
'multipoint',
((10, 40), (40, 30), (20, 20), (30, 10)),
'MULTIPOINT ((10 40), (40 30), (20 20), (30 10))',
),
(
'multilinestring',
(
((10, 10), (20, 20), (10, 40)),
((40, 40), (30, 30), (40, 20), (30, 10)),
),
(
'MULTILINESTRING ((10 10, 20 20, 10 40), '
+ '(40 40, 30 30, 40 20, 30 10))'
),
),
(
'multipolygon',
(
(((40, 40), (20, 45), (45, 30), (40, 40)),),
(
(
(20, 35),
(10, 30),
(10, 10),
(30, 5),
(45, 20),
(20, 35),
),
((30, 20), (20, 15), (20, 25), (30, 20)),
),
),
(
'MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), '
+ '((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), '
+ '(30 20, 20 15, 20 25, 30 20)))'
),
),
],
)
def test_geo_literals_smoke(backend, shape, value, modifier, expected):
"""Smoke tests for geo spatial literals"""
expr_type = '{}{}{}'.format(
shape,
';{}'.format(modifier['srid']) if 'srid' in modifier else '',
':{}'.format(modifier['geo_type']) if 'geo_type' in modifier else '',
)
polygon2 = (
((10, 10), (14, 10), (14, 14), (10, 14), (10, 10)),
((11, 11), (12, 11), (12, 12), (11, 12), (11, 11)),
expr = ibis.literal(value, type=expr_type).name('tmp')
result_expected = "SELECT '{}{}'{}".format(
'SRID={};'.format(modifier['srid']) if 'srid' in modifier else '',
expected,
'::{}'.format(modifier['geo_type']) if 'geo_type' in modifier else '',
)
multipolygon = [polygon1, polygon2]
ibis.literal(multipolygon, type='multipolygon{}'.format(modifier))

assert str(backend.compile(expr)) == result_expected

def test_geo_ops_smoke(geo_table):
"""Smoke tests for geo spatial operations."""
t = geo_table

# alias for fields
point = t.geo_point
linestring = t.geo_linestring
polygon = t.geo_polygon
multipolygon = t.geo_multipolygon

# test ops
point.srid()
point.set_srid(4326)
point.x()
point.y()

linestring.contains(point)
linestring.end_point()
linestring.length()
linestring.max_distance(point)
linestring.point_n(1)
linestring.start_point()
linestring.x_max()
linestring.x_min()
linestring.y_max()
linestring.y_min()

polygon.area()
polygon.perimeter()

multipolygon.n_points()
multipolygon.n_rings()
@pytest.mark.parametrize(
'fn_expr',
[
pytest.param(lambda t: t.geo_point.srid(), id='point_srid'),
pytest.param(
lambda t: t.geo_point.set_srid(4326), id='point_set_srid'
),
pytest.param(lambda t: t.geo_point.x(), id='point_x'),
pytest.param(lambda t: t.geo_point.y(), id='point_y'),
pytest.param(
lambda t: t.geo_linestring.contains(t.geo_point),
id='linestring_contains',
),
pytest.param(
lambda t: t.geo_linestring.end_point(), id='linestring_end_point'
),
pytest.param(
lambda t: t.geo_linestring.length(), id='linestring_length'
),
pytest.param(
lambda t: t.geo_linestring.max_distance(t.geo_point),
id='linestring_max_distance',
),
pytest.param(
lambda t: t.geo_linestring.point_n(1), id='linestring_point_n'
),
pytest.param(
lambda t: t.geo_linestring.start_point(),
id='linestring_start_point',
),
pytest.param(
lambda t: t.geo_linestring.x_max(), id='linestring_x_max'
),
pytest.param(
lambda t: t.geo_linestring.x_min(), id='linestring_x_min'
),
pytest.param(
lambda t: t.geo_linestring.y_max(), id='linestring_y_max'
),
pytest.param(
lambda t: t.geo_linestring.y_min(), id='linestring_y_min'
),
pytest.param(lambda t: t.geo_polygon.area(), id='polygon_area'),
pytest.param(
lambda t: t.geo_polygon.perimeter(), id='polygon_perimeter'
),
pytest.param(
lambda t: t.geo_multipolygon.n_points(), id='multipolygon_n_points'
),
pytest.param(
lambda t: t.geo_multipolygon.n_rings(), id='multipolygon_n_rings'
),
# TODO: the mock tests don't support multipoint and multilinestring
# yet, but once they do, add some more tests here.
],
)
@pytest.mark.parametrize('backend', [mock_omniscidb, mock_postgis])
@pytest.mark.xfail_unsupported
def test_geo_ops_smoke(backend, fn_expr):
"""Smoke tests for geo spatial operations."""
geo_table = backend.table('geo')
assert fn_expr(geo_table).compile() != ''
20 changes: 19 additions & 1 deletion ibis/expr/tests/test_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import ibis.expr.operations as ops
import ibis.expr.rules as rlz
import ibis.expr.types as ir
from ibis.common import IbisTypeError
from ibis.common.exceptions import IbisTypeError
from ibis.expr.signature import Argument as Arg


Expand Down Expand Up @@ -110,3 +110,21 @@ def output_type(self):
node = SpecialTable('foo', ibis.schema([('a', 'int64')]), con)
expr = node.to_expr()
assert isinstance(expr, MyTableExpr)


@pytest.fixture(scope='session')
def dummy_op():
class DummyOp(ops.ValueOp):
arg = Arg(rlz.any)

return DummyOp


def test_too_many_args_not_allowed(dummy_op):
with pytest.raises(TypeError):
dummy_op(1, 2)


def test_too_few_args_not_allowed(dummy_op):
with pytest.raises(TypeError):
dummy_op()
2 changes: 1 addition & 1 deletion ibis/expr/tests/test_rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import ibis.expr.datatypes as dt
import ibis.expr.rules as rlz
import ibis.expr.types as ir
from ibis.common import IbisTypeError
from ibis.common.exceptions import IbisTypeError

table = ibis.table(
[('int_col', 'int64'), ('string_col', 'string'), ('double_col', 'double')]
Expand Down
2 changes: 1 addition & 1 deletion ibis/expr/tests/test_signature.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pytest
from toolz import identity

from ibis.common import IbisTypeError
from ibis.common.exceptions import IbisTypeError
from ibis.expr.signature import Annotable, Argument, TypeSignature


Expand Down
49 changes: 42 additions & 7 deletions ibis/expr/tests/test_table.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
import datetime
import pickle
import re

import pandas as pd
import pytest

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.config as config
import ibis.expr.analysis as L
import ibis.expr.api as api
import ibis.expr.datatypes as dt
import ibis.expr.operations as ops
import ibis.expr.types as ir
from ibis.common import ExpressionError, RelationError
from ibis.common.exceptions import ExpressionError, RelationError
from ibis.expr.types import ColumnExpr, TableExpr
from ibis.tests.util import assert_equal

Expand Down Expand Up @@ -695,6 +697,42 @@ def test_asof_join_with_by():
assert by.left.op().name == by.right.op().name == 'key'


@pytest.mark.parametrize(
('ibis_interval', 'timedelta_interval'),
[
[ibis.interval(days=2), pd.Timedelta('2 days')],
[ibis.interval(days=2), datetime.timedelta(days=2)],
[ibis.interval(hours=5), pd.Timedelta('5 hours')],
[ibis.interval(hours=5), datetime.timedelta(hours=5)],
[ibis.interval(minutes=7), pd.Timedelta('7 minutes')],
[ibis.interval(minutes=7), datetime.timedelta(minutes=7)],
[ibis.interval(seconds=9), pd.Timedelta('9 seconds')],
[ibis.interval(seconds=9), datetime.timedelta(seconds=9)],
[ibis.interval(milliseconds=11), pd.Timedelta('11 milliseconds')],
[ibis.interval(milliseconds=11), datetime.timedelta(milliseconds=11)],
[ibis.interval(microseconds=15), pd.Timedelta('15 microseconds')],
[ibis.interval(microseconds=15), datetime.timedelta(microseconds=15)],
[ibis.interval(nanoseconds=17), pd.Timedelta('17 nanoseconds')],
],
)
def test_asof_join_with_tolerance(ibis_interval, timedelta_interval):
left = ibis.table(
[('time', 'int32'), ('key', 'int32'), ('value', 'double')]
)
right = ibis.table(
[('time', 'int32'), ('key', 'int32'), ('value2', 'double')]
)

joined = api.asof_join(left, right, 'time', tolerance=ibis_interval)
tolerance = joined.op().tolerance
assert_equal(tolerance, ibis_interval)

joined = api.asof_join(left, right, 'time', tolerance=timedelta_interval)
tolerance = joined.op().tolerance
assert isinstance(tolerance, ir.IntervalScalar)
assert isinstance(tolerance.op(), ops.Literal)


def test_equijoin_schema_merge():
table1 = ibis.table([('key1', 'string'), ('value1', 'double')])
table2 = ibis.table([('key2', 'string'), ('stuff', 'int32')])
Expand Down Expand Up @@ -938,12 +976,9 @@ def test_join_invalid_expr_type(con):
invalid_right = left.foo_id
join_key = ['bar_id']

with pytest.raises(TypeError) as e:
with pytest.raises(TypeError, match=type(invalid_right).__name__):
left.inner_join(invalid_right, join_key)

message = str(e)
assert type(invalid_right).__name__ in message


def test_join_non_boolean_expr(con):
t1 = con.table('star1')
Expand Down Expand Up @@ -1067,7 +1102,7 @@ def test_cannot_use_existence_expression_in_join(table):


def test_not_exists_predicate(t1, t2):
cond = -(t1.key1 == t2.key1).any()
cond = -((t1.key1 == t2.key1).any())
assert isinstance(cond.op(), ops.NotAny)


Expand Down
7 changes: 6 additions & 1 deletion ibis/expr/tests/test_timestamp.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import numpy as np
import pandas as pd
import pytest
from pkg_resources import parse_version

import ibis
import ibis.expr.api as api
Expand Down Expand Up @@ -98,7 +99,11 @@ def test_comparisons_pandas_timestamp(alltypes):
assert isinstance(op.right, ir.TimestampScalar)


@pytest.mark.xfail(raises=TypeError, reason='Upstream pandas bug')
@pytest.mark.xfail(
condition=parse_version(pd.__version__) < parse_version('0.25'),
raises=TypeError,
reason='Upstream pandas bug',
)
def test_greater_comparison_pandas_timestamp(alltypes):
val = pd.Timestamp('2015-01-01 00:00:00')
expr2 = val < alltypes.i
Expand Down
16 changes: 11 additions & 5 deletions ibis/expr/tests/test_value_exprs.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@
import toolz

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.analysis as L
import ibis.expr.api as api
import ibis.expr.datatypes as dt
import ibis.expr.operations as ops
import ibis.expr.rules as rlz
import ibis.expr.types as ir
from ibis import literal
from ibis.common import IbisTypeError
from ibis.common.exceptions import IbisTypeError
from ibis.expr.signature import Argument as Arg
from ibis.tests.util import assert_equal

Expand Down Expand Up @@ -84,6 +84,8 @@ def test_literal_with_implicit_type(value, expected_type):
lineCA = [pointC, pointA]
polygon1 = [lineAB, lineBC, lineCA]
polygon2 = [lineAB, lineBC, lineCA]
multilinestring = [lineAB, lineBC, lineCA]
multipoint = [pointA, pointB, pointC]
multipolygon1 = [polygon1, polygon2]


Expand All @@ -109,6 +111,10 @@ def test_literal_with_implicit_type(value, expected_type):
(tuple(lineAB), 'linestring'),
(list(polygon1), 'polygon'),
(tuple(polygon1), 'polygon'),
(list(multilinestring), 'multilinestring'),
(tuple(multilinestring), 'multilinestring'),
(list(multipoint), 'multipoint'),
(tuple(multipoint), 'multipoint'),
(list(multipolygon1), 'multipolygon'),
(tuple(multipolygon1), 'multipolygon'),
],
Expand Down Expand Up @@ -574,7 +580,7 @@ def test_negate(table, col):


def test_negate_boolean_scalar():
result = -ibis.literal(False)
result = -(ibis.literal(False))
assert isinstance(result, ir.BooleanScalar)
assert isinstance(result.op(), ops.Negate)

Expand Down Expand Up @@ -700,7 +706,7 @@ def test_null_column():
def test_null_column_union():
s = ibis.table([('a', 'string'), ('b', 'double')])
t = ibis.table([('a', 'string')])
with pytest.raises(ibis.common.RelationError):
with pytest.raises(ibis.common.exceptions.RelationError):
s.union(t.mutate(b=ibis.NA)) # needs a type
assert s.union(t.mutate(b=ibis.NA.cast('double'))).schema() == s.schema()

Expand Down Expand Up @@ -787,7 +793,7 @@ def test_binop_string_type_error(table, operation):
(operator.mul, 'a', 0, 'int8'),
(operator.mul, 'a', 5, 'int16'),
(operator.mul, 'a', 2 ** 24, 'int32'),
(operator.mul, 'a', -2 ** 24 + 1, 'int32'),
(operator.mul, 'a', -(2 ** 24) + 1, 'int32'),
(operator.mul, 'a', 1.5, 'double'),
(operator.mul, 'b', 0, 'int16'),
(operator.mul, 'b', 5, 'int32'),
Expand Down
7 changes: 7 additions & 0 deletions ibis/expr/tests/test_visualize.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,10 @@ def test_asof_join():
result = joined[left, right.foo]
graph = viz.to_graph(result)
assert key(result) in graph.source


def test_html_escape():
# Check that we correctly escape HTML <> characters in the graphviz
# representation. If an error is thrown, _repr_png_ returns None.
expr = ibis.table([('<a & b>', ibis.expr.datatypes.Array('string'))])
assert expr._repr_png_() is not None
112 changes: 55 additions & 57 deletions ibis/expr/tests/test_window_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import pytest

import ibis
import ibis.common as com
import ibis.common.exceptions as com
from ibis.expr.window import _determine_how, rows_with_max_lookback
from ibis.tests.util import assert_equal

Expand Down Expand Up @@ -61,45 +61,68 @@ def test_combine_windows(alltypes):

# Cannot combine windows of varying types.
w6 = ibis.range_window(preceding=5, following=5)
with pytest.raises(ibis.common.IbisInputError):
with pytest.raises(ibis.common.exceptions.IbisInputError):
w1.combine(w6)

w7 = ibis.trailing_window(
rows_with_max_lookback(3, ibis.interval(days=5))

def test_combine_windows_with_zero_offset():
w1 = ibis.window(preceding=0, following=5)
w2 = ibis.window(preceding=7, following=10)
w3 = w1.combine(w2)
expected = ibis.window(preceding=7, following=5)
assert_equal(w3, expected)

w4 = ibis.window(preceding=3, following=0)
w5 = w4.combine(w2)
expected = ibis.window(preceding=3, following=10)
assert_equal(w5, expected)


def test_combine_window_with_interval_offset(alltypes):
t = alltypes
w1 = ibis.trailing_range_window(
preceding=ibis.interval(days=3), order_by=t.e
)
w8 = ibis.trailing_window(
rows_with_max_lookback(5, ibis.interval(days=7))
w2 = ibis.trailing_range_window(
preceding=ibis.interval(days=4), order_by=t.f
)
w3 = w1.combine(w2)
expected = ibis.trailing_range_window(
preceding=ibis.interval(days=3), order_by=[t.e, t.f]
)
assert_equal(w3, expected)

w4 = ibis.range_window(following=ibis.interval(days=5), order_by=t.e)
w5 = ibis.range_window(following=ibis.interval(days=7), order_by=t.f)
expected = ibis.range_window(
following=ibis.interval(days=5), order_by=[t.e, t.f]
)
w9 = w7.combine(w8)
w6 = w4.combine(w5)
assert_equal(w6, expected)


def test_combine_window_with_max_lookback():
w1 = ibis.trailing_window(rows_with_max_lookback(3, ibis.interval(days=5)))
w2 = ibis.trailing_window(rows_with_max_lookback(5, ibis.interval(days=7)))
w3 = w1.combine(w2)
expected = ibis.trailing_window(
rows_with_max_lookback(3, ibis.interval(days=5))
)
assert_equal(w9, expected)
assert_equal(w3, expected)


def test_replace_window(alltypes):
t = alltypes
w1 = ibis.window(
preceding=5,
following=1,
group_by=t.a,
order_by=t.b
)
w1 = ibis.window(preceding=5, following=1, group_by=t.a, order_by=t.b)
w2 = w1.group_by(t.c)
expected = ibis.window(
preceding=5,
following=1,
group_by=[t.a, t.c],
order_by=t.b
preceding=5, following=1, group_by=[t.a, t.c], order_by=t.b
)
assert_equal(w2, expected)

w3 = w1.order_by(t.d)
expected = ibis.window(
preceding=5,
following=1,
group_by=t.a,
order_by=[t.b, t.d]
preceding=5, following=1, group_by=t.a, order_by=[t.b, t.d]
)
assert_equal(w3, expected)

Expand All @@ -108,8 +131,7 @@ def test_replace_window(alltypes):
)
w5 = w4.group_by(t.a)
expected = ibis.trailing_window(
rows_with_max_lookback(3, ibis.interval(months=3)),
group_by=t.a
rows_with_max_lookback(3, ibis.interval(months=3)), group_by=t.a
)
assert_equal(w5, expected)

Expand Down Expand Up @@ -245,60 +267,36 @@ def test_max_rows_with_lookback_validate(alltypes):

def test_window_equals(alltypes):
t = alltypes
w1 = ibis.window(
preceding=1,
following=2,
group_by=t.a,
order_by=t.b
)
w2 = ibis.window(
preceding=1,
following=2,
group_by=t.a,
order_by=t.b
)
w1 = ibis.window(preceding=1, following=2, group_by=t.a, order_by=t.b)
w2 = ibis.window(preceding=1, following=2, group_by=t.a, order_by=t.b)
assert w1.equals(w2)

w3 = ibis.window(
preceding=1,
following=2,
group_by=t.a,
order_by=t.c
)
w3 = ibis.window(preceding=1, following=2, group_by=t.a, order_by=t.c)
assert not w1.equals(w3)

w4 = ibis.range_window(
preceding=ibis.interval(hours=3),
group_by=t.d
)
w5 = ibis.range_window(
preceding=ibis.interval(hours=3),
group_by=t.d
)
w4 = ibis.range_window(preceding=ibis.interval(hours=3), group_by=t.d)
w5 = ibis.range_window(preceding=ibis.interval(hours=3), group_by=t.d)
assert w4.equals(w5)

w6 = ibis.range_window(
preceding=ibis.interval(hours=1),
group_by=t.d
)
w6 = ibis.range_window(preceding=ibis.interval(hours=1), group_by=t.d)
assert not w4.equals(w6)

w7 = ibis.trailing_window(
rows_with_max_lookback(3, ibis.interval(days=5)),
group_by=t.a,
order_by=t.b
order_by=t.b,
)
w8 = ibis.trailing_window(
rows_with_max_lookback(3, ibis.interval(days=5)),
group_by=t.a,
order_by=t.b
order_by=t.b,
)
assert w7.equals(w8)

w9 = ibis.trailing_window(
rows_with_max_lookback(3, ibis.interval(months=5)),
group_by=t.a,
order_by=t.b
order_by=t.b,
)
assert not w7.equals(w9)

Expand Down
66 changes: 65 additions & 1 deletion ibis/expr/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import numpy as np

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.config as config
import ibis.util as util

Expand Down Expand Up @@ -794,6 +794,30 @@ class MapColumn(AnyColumn, MapValue):
pass # noqa: E701,E302


class JSONValue(StringValue):
pass # noqa: E701,E302


class JSONScalar(StringScalar, JSONValue):
pass # noqa: E701,E302


class JSONColumn(StringColumn, JSONValue):
pass # noqa: E701,E302


class JSONBValue(BinaryValue):
pass # noqa: E701,E302


class JSONBScalar(BinaryScalar, JSONBValue):
pass # noqa: E701,E302


class JSONBColumn(BinaryColumn, JSONBValue):
pass # noqa: E701,E302


class StructValue(AnyValue):
def __dir__(self):
return sorted(
Expand Down Expand Up @@ -869,6 +893,34 @@ class PolygonColumn(GeoSpatialColumn, PolygonValue):
pass # noqa: E701,E302


class MultiLineStringValue(GeoSpatialValue):
pass # noqa: E701,E302


class MultiLineStringScalar(
GeoSpatialScalar, MultiLineStringValue
): # noqa: E302
pass # noqa: E701


class MultiLineStringColumn(
GeoSpatialColumn, MultiLineStringValue
): # noqa: E302
pass # noqa: E701


class MultiPointValue(GeoSpatialValue):
pass # noqa: E701,E302


class MultiPointScalar(GeoSpatialScalar, MultiPointValue): # noqa: E302
pass # noqa: E701


class MultiPointColumn(GeoSpatialColumn, MultiPointValue): # noqa: E302
pass # noqa: E701


class MultiPolygonValue(GeoSpatialValue):
pass # noqa: E701,E302

Expand All @@ -881,6 +933,18 @@ class MultiPolygonColumn(GeoSpatialColumn, MultiPolygonValue): # noqa: E302
pass # noqa: E701


class UUIDValue(StringValue):
pass # noqa: E701,E302


class UUIDScalar(StringScalar, UUIDValue):
pass # noqa: E701,E302


class UUIDColumn(StringColumn, UUIDValue):
pass # noqa: E701,E302


class ListExpr(ColumnExpr, AnyValue):
@property
def values(self):
Expand Down
13 changes: 8 additions & 5 deletions ibis/expr/visualize.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import tempfile
from html import escape

import graphviz as gv

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.operations as ops
import ibis.expr.types as ir

Expand Down Expand Up @@ -42,7 +43,9 @@ def get_type(expr):

return (
''.join(
'<BR ALIGN="LEFT" /> <I>{}</I>: {}'.format(name, type)
'<BR ALIGN="LEFT" /> <I>{}</I>: {}'.format(
escape(name), escape(str(type))
)
for name, type in zip(schema.names, schema.types)
)
+ '<BR ALIGN="LEFT" />'
Expand All @@ -53,21 +56,21 @@ def get_label(expr, argname=None):
import ibis.expr.operations as ops

node = expr.op()
typename = get_type(expr)
typename = get_type(expr) # Already an escaped string
name = type(node).__name__
nodename = getattr(node, 'name', argname)
if nodename is not None:
if isinstance(node, ops.TableNode):
label_fmt = '<<I>{}</I>: <B>{}</B>{}>'
else:
label_fmt = '<<I>{}</I>: <B>{}</B> \u27f6 {}>'
label = label_fmt.format(nodename, name, typename)
label = label_fmt.format(escape(nodename), escape(name), typename)
else:
if isinstance(node, ops.TableNode):
label_fmt = '<<B>{}</B>{}>'
else:
label_fmt = '<<B>{}</B> \u27f6 {}>'
label = label_fmt.format(name, typename)
label = label_fmt.format(escape(name), typename)
return label


Expand Down
60 changes: 38 additions & 22 deletions ibis/expr/window.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import numpy as np
import pandas as pd

import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.operations as ops
import ibis.expr.types as ir
import ibis.util as util
Expand All @@ -16,10 +16,20 @@ def _sequence_to_tuple(x):
return tuple(x) if util.is_iterable(x) else x


RowsWithMaxLookback = NamedTuple('RowsWithMaxLookback',
[('rows', Union[int, np.integer]),
('max_lookback', ir.IntervalValue)]
)
RowsWithMaxLookback = NamedTuple(
'RowsWithMaxLookback',
[('rows', Union[int, np.integer]), ('max_lookback', ir.IntervalValue)],
)


def _choose_non_empty_val(first, second):
if isinstance(first, (int, np.integer)) and first:
non_empty_value = first
elif not isinstance(first, (int, np.integer)) and first is not None:
non_empty_value = first
else:
non_empty_value = second
return non_empty_value


def _determine_how(preceding):
Expand Down Expand Up @@ -65,8 +75,10 @@ def _get_preceding_value_simple(preceding):
def _get_preceding_value_mlb(preceding):
preceding_value = preceding.rows
if not isinstance(preceding_value, (int, np.integer)):
raise TypeError("'Rows with max look-back' only supports integer "
"row-based indexing.")
raise TypeError(
"'Rows with max look-back' only supports integer "
"row-based indexing."
)
return preceding_value


Expand Down Expand Up @@ -108,8 +120,10 @@ def __init__(
self._order_by.append(x)

if isinstance(preceding, RowsWithMaxLookback):
self.preceding = preceding.rows
self.max_lookback = preceding.max_lookback
# the offset interval is used as the 'preceding' value of a window
# while 'rows' is used to adjust the window created using offset
self.preceding = preceding.max_lookback
self.max_lookback = preceding.rows
else:
self.preceding = _sequence_to_tuple(preceding)
self.max_lookback = max_lookback
Expand Down Expand Up @@ -207,7 +221,8 @@ def _validate_frame(self):

if self.max_lookback is not None:
if not isinstance(
self.max_lookback, (ir.IntervalValue, pd.Timedelta)):
self.preceding, (ir.IntervalValue, pd.Timedelta)
):
raise com.IbisInputError(
"'max_lookback' must be specified as an interval "
"or pandas.Timedelta object"
Expand All @@ -228,11 +243,11 @@ def combine(self, window):
"Expecting '{}' Window, got '{}'"
).format(self.how.upper(), window.how.upper())
)
mlb = self.max_lookback

kwds = dict(
preceding=self.preceding or window.preceding,
following=self.following or window.following,
max_lookback=mlb if mlb is not None else window.max_lookback,
preceding=_choose_non_empty_val(self.preceding, window.preceding),
following=_choose_non_empty_val(self.following, window.following),
max_lookback=self.max_lookback or window.max_lookback,
group_by=self._group_by + window._group_by,
order_by=self._order_by + window._order_by,
)
Expand Down Expand Up @@ -286,12 +301,12 @@ def equals(self, other, cache=None):
cache[self, other] = False
return False

equal = ops.all_equal(
self.preceding, other.preceding, cache=cache
) and ops.all_equal(
self.following, other.following, cache=cache
) and ops.all_equal(
self.max_lookback, other.max_lookback, cache=cache
equal = (
ops.all_equal(self.preceding, other.preceding, cache=cache)
and ops.all_equal(self.following, other.following, cache=cache)
and ops.all_equal(
self.max_lookback, other.max_lookback, cache=cache
)
)
cache[self, other] = equal
return equal
Expand Down Expand Up @@ -405,7 +420,8 @@ def trailing_window(preceding, group_by=None, order_by=None):
preceding : int, float or expression of intervals, i.e.
ibis.interval(days=1) + ibis.interval(hours=5)
Int indicates number of trailing rows to include;
0 includes only the current row.
0 includes only the current row, 1 includes the current row and one
preceding row.
Interval indicates a trailing range window.
group_by : expressions, default None
Either specify here or with TableExpr.group_by
Expand All @@ -424,7 +440,7 @@ def trailing_window(preceding, group_by=None, order_by=None):
following=0,
group_by=group_by,
order_by=order_by,
how=how
how=how,
)


Expand Down
2 changes: 1 addition & 1 deletion ibis/file/csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def table(self, name, path=None, schema=None, **kwargs):
sample = _read_csv(f, schema=schema, header=0, nrows=50, **kwargs)

# infer sample's schema and define table
schema = sch.infer(sample)
schema = sch.infer(sample, schema=schema)
table = self.table_class(name, schema, self, **kwargs).to_expr()

self.dictionary[name] = f
Expand Down
7 changes: 5 additions & 2 deletions ibis/file/parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@
'string': dt.String,
'binary': dt.Binary,
'bool': dt.Boolean,
'timestamp[ns]': dt.Timestamp,
'timestamp[us]': dt.Timestamp,
}


Expand All @@ -39,6 +37,11 @@ def pa_dtype(arrow_type, nullable=True):
return _arrow_dtypes[str(arrow_type)](nullable=nullable)


@dt.dtype.register(pa.lib.TimestampType)
def pa_timestamp_type(arrow_type, nullable=True):
return dt.Timestamp(arrow_type.tz, nullable=nullable)


@sch.infer.register(pq.ParquetSchema)
def infer_parquet_schema(schema):
pairs = []
Expand Down
3 changes: 3 additions & 0 deletions ibis/file/tests/test_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@
(pa.float64(), dt.float64),
(pa.string(), dt.string),
(pa.timestamp('ns'), dt.timestamp),
(pa.timestamp('us'), dt.timestamp),
(pa.timestamp('ns', 'UTC'), dt.Timestamp('UTC')),
(pa.timestamp('us', 'Europe/Paris'), dt.Timestamp('Europe/Paris')),
],
ids=lambda x: str(x),
)
Expand Down
446 changes: 380 additions & 66 deletions ibis/filesystems.py

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion ibis/impala/api.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import ibis.common as com
import ibis.common.exceptions as com
from ibis.config import options

# these objects are exposed in the public API and are not used in the module
from ibis.impala.client import ( # noqa: F401
ImpalaClient,
Expand Down
6 changes: 3 additions & 3 deletions ibis/impala/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import pandas as pd
from pkg_resources import parse_version

import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.expr.operations as ops
import ibis.expr.rules as rlz
Expand Down Expand Up @@ -1139,7 +1139,7 @@ def create_table(
obj : TableExpr or pandas.DataFrame, optional
If passed, creates table from select statement results
schema : ibis.Schema, optional
Mutually exclusive with expr, creates an empty table with a
Mutually exclusive with obj, creates an empty table with a
particular schema
database : string, default None (optional)
force : boolean, default False
Expand Down Expand Up @@ -1196,7 +1196,7 @@ def create_table(
partition=partition,
)
else:
raise com.IbisError('Must pass expr or schema')
raise com.IbisError('Must pass obj or schema')

return self._execute(statement)

Expand Down
31 changes: 21 additions & 10 deletions ibis/impala/compiler.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import datetime
import itertools
import math
from io import StringIO
from operator import add, mul, sub
from typing import Optional

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.analysis as L
import ibis.expr.datatypes as dt
import ibis.expr.operations as ops
Expand Down Expand Up @@ -335,8 +336,10 @@ def _format_window(translator, op, window):
components = []

if window.max_lookback is not None:
raise NotImplementedError('Rows with max lookback is not implemented '
'for Impala-based backends.')
raise NotImplementedError(
'Rows with max lookback is not implemented '
'for Impala-based backends.'
)

if len(window._group_by) > 0:
partition_args = [translator.translate(x) for x in window._group_by]
Expand Down Expand Up @@ -482,7 +485,7 @@ def _negate(translator, expr):


def _not(translator, expr):
arg, = expr.op().args
(arg,) = expr.op().args
formatted_arg = translator.translate(arg)
if _needs_parens(arg):
formatted_arg = _parenthesize(formatted_arg)
Expand Down Expand Up @@ -631,10 +634,18 @@ def _string_literal_format(translator, expr):

def _number_literal_format(translator, expr):
value = expr.op().value
formatted = repr(value)

if formatted in {'nan', 'inf', '-inf'}:
return "CAST({!r} AS DOUBLE)".format(formatted)
if math.isfinite(value):
formatted = repr(value)
else:
if math.isnan(value):
formatted_val = 'NaN'
elif math.isinf(value):
if value > 0:
formatted_val = 'Infinity'
else:
formatted_val = '-Infinity'
formatted = "CAST({!r} AS DOUBLE)".format(formatted_val)

return formatted

Expand Down Expand Up @@ -842,12 +853,12 @@ def extract_field_formatter(translator, expr):


def _day_of_week_index(t, expr):
arg, = expr.op().args
(arg,) = expr.op().args
return 'pmod(dayofweek({}) - 2, 7)'.format(t.translate(arg))


def _day_of_week_name(t, expr):
arg, = expr.op().args
(arg,) = expr.op().args
return 'dayname({})'.format(t.translate(arg))


Expand Down Expand Up @@ -1119,7 +1130,7 @@ def _string_like(translator, expr):


def _sign(translator, expr):
arg, = expr.op().args
(arg,) = expr.op().args
translated_arg = translator.translate(arg)
translated_type = _type_to_sql_string(expr.type())
if expr.type() != dt.float:
Expand Down
4 changes: 2 additions & 2 deletions ibis/impala/ddl.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ def __init__(
example_table=None,
schema=None,
external=True,
**kwargs
**kwargs,
):
super().__init__(
table_name,
Expand Down Expand Up @@ -361,7 +361,7 @@ def __init__(
lineterminator=None,
na_rep=None,
external=True,
**kwargs
**kwargs,
):
table_format = DelimitedFormat(
path,
Expand Down
4 changes: 2 additions & 2 deletions ibis/impala/kudu_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import pandas as pd

import ibis.expr.datatypes as dt
from ibis.common import IbisError
from ibis.common.exceptions import IbisError
from ibis.expr.api import schema
from ibis.impala import ddl

Expand Down Expand Up @@ -238,7 +238,7 @@ def __init__(
schema,
key_columns,
external=True,
**kwargs
**kwargs,
):
self.kudu_table_name = kudu_table_name
self.master_addrs = master_addrs
Expand Down
2 changes: 1 addition & 1 deletion ibis/impala/pandas_interop.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import tempfile
from posixpath import join as pjoin

import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.schema as sch
import ibis.util as util
from ibis.config import options
Expand Down
4 changes: 2 additions & 2 deletions ibis/impala/tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import pytz

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.config as config
import ibis.expr.datatypes as dt
import ibis.expr.types as ir
Expand Down Expand Up @@ -185,7 +185,7 @@ def test_verbose_log_queries(con, test_data_db):
con.table('tpch_orders', database=test_data_db)

assert len(queries) == 1
query, = queries
(query,) = queries
expected = 'DESCRIBE {}.`tpch_orders`'.format(test_data_db)
assert query == expected

Expand Down
2 changes: 1 addition & 1 deletion ibis/impala/tests/test_ddl.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pytest

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.expr.types as ir
import ibis.util as util
Expand Down
18 changes: 9 additions & 9 deletions ibis/impala/tests/test_exprs.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import ibis.expr.api as api
import ibis.expr.types as ir
from ibis import literal as L
from ibis.common import RelationError
from ibis.common.exceptions import RelationError
from ibis.expr.datatypes import Category
from ibis.expr.tests.mocks import MockConnection
from ibis.impala.compiler import ( # noqa: E402
Expand Down Expand Up @@ -209,9 +209,9 @@ def test_decimal_casts(self):

def test_negate(self):
cases = [
(-self.table['a'], '-`a`'),
(-self.table['f'], '-`f`'),
(-self.table['h'], 'NOT `h`'),
(-(self.table['a']), '-`a`'),
(-(self.table['f']), '-`f`'),
(-(self.table['h']), 'NOT `h`'),
]
self._check_expr_cases(cases)

Expand Down Expand Up @@ -357,10 +357,10 @@ def test_any_all(self):
bool_expr = t.f == 0

cases = [
(bool_expr.any(), 'sum(`f` = 0) > 0'),
(-bool_expr.any(), 'sum(`f` = 0) = 0'),
(bool_expr.all(), 'sum(`f` = 0) = count(*)'),
(-bool_expr.all(), 'sum(`f` = 0) < count(*)'),
(bool_expr.any(), 'max(`f` = 0)'),
(-(bool_expr.any()), 'max(`f` = 0) = FALSE'),
(bool_expr.all(), 'min(`f` = 0)'),
(-(bool_expr.all()), 'min(`f` = 0) = FALSE'),
]
self._check_expr_cases(cases)

Expand Down Expand Up @@ -1574,7 +1574,7 @@ def test_analytic_functions(alltypes):
def test_anti_join_self_reference_works(con, alltypes):
t = alltypes.limit(100)
t2 = t.view()
case = t[-(t.string_col == t2.string_col).any()]
case = t[-((t.string_col == t2.string_col).any())]
con.explain(case)


Expand Down
4 changes: 1 addition & 3 deletions ibis/impala/tests/test_kudu_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,7 @@ def __init__(self):
super().__init__()

# band-aid until Kudu support merged into Impala mainline
self.test_host = os.getenv(
'IBIS_TEST_KIMPALA_HOST', 'impala'
)
self.test_host = os.getenv('IBIS_TEST_KIMPALA_HOST', 'impala')

# XXX
self.impala_host = self.test_host
Expand Down
6 changes: 3 additions & 3 deletions ibis/impala/tests/test_udf.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@
import pytest

import ibis
import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.expr.rules as rules
import ibis.expr.types as ir
import ibis.impala as api # noqa: E402
import ibis.util as util
from ibis.common import IbisTypeError
from ibis.common.exceptions import IbisTypeError
from ibis.expr.tests.mocks import MockConnection
from ibis.impala import ddl # noqa: E402

Expand Down Expand Up @@ -572,7 +572,7 @@ def make_ex(serialize=False):
"\nmerge_fn='Merge'"
)
+ serialize
+ ("\nfinalize_fn='Finalize'")
+ "\nfinalize_fn='Finalize'"
)

for ser in [True, False]:
Expand Down
4 changes: 2 additions & 2 deletions ibis/impala/tests/test_window.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import pytest

import ibis
import ibis.common as com
import ibis.common.exceptions as com
from ibis import window
from ibis.expr.window import rows_with_max_lookback
from ibis.impala.compiler import to_sql # noqa: E402
Expand Down Expand Up @@ -107,7 +107,7 @@ def test_add_default_order_by(alltypes):
(
ibis.trailing_window(10),
'rows between 10 preceding and current row',
)
),
],
)
def test_window_frame_specs(con, window, frame):
Expand Down
2 changes: 1 addition & 1 deletion ibis/impala/udf.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

import re

import ibis.common as com
import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.expr.operations as ops
import ibis.expr.rules as rlz
Expand Down
77 changes: 0 additions & 77 deletions ibis/mapd/api.py

This file was deleted.

266 changes: 0 additions & 266 deletions ibis/mapd/compiler.py

This file was deleted.

Loading