Implement type inference. Remove DateType and DateColumn. #210.

CHANGELOG

@@ -1,7 +1,8 @@
 0.8.0
 -----
 
-
+* Remove DateType and DateColumn.
+* Implement robust type inference. (#210)
 
 0.7.0
 -----

agate/aggregations.py

@@ -19,9 +19,9 @@
 from collections import defaultdict
 import datetime
 
-from agate.column_types import BooleanType, NumberType
-from agate.columns import BooleanColumn, DateColumn, DateTimeColumn, NumberColumn, TextColumn
-from agate.exceptions import NullCalculationError, UnsupportedAggregationError
+from agate.column_types import *
+from agate.columns import *
+from agate.exceptions import *
 
 class Aggregation(object): #pragma: no cover
     """
@@ -146,12 +146,10 @@ def run(self, column):
 class Min(Aggregation):
     """
     Compute the minimum value in a column. May be applied to
-    :class:`.DateColumn`, :class:`.DateTimeColumn` and :class:`.NumberColumn`.
+    :class:`.DateTimeColumn` and :class:`.NumberColumn`.
     """
     def get_aggregate_column_type(self, column):
-        if isinstance(column, DateColumn):
-            return DateType()
-        elif isinstance(column, DateTimeColumn):
+        if isinstance(column, DateTimeColumn):
             return DateTimeType()
         elif isinstance(column, NumberColumn):
             return NumberType()
@@ -162,22 +160,18 @@ def run(self, column):
         """
         :returns: :class:`datetime.date`
         """
-        supported_columns = (DateColumn, DateTimeColumn, NumberColumn)
-
-        if not any(isinstance(column, t) for t in supported_columns):
+        if not (isinstance(column, DateTimeColumn) or isinstance(column, NumberColumn)):
             raise UnsupportedAggregationError(self, column)
 
         return min(column.get_data_without_nulls())
 
 class Max(Aggregation):
     """
     Compute the maximum value in a column. May be applied to
-    :class:`.DateColumn`, :class:`.DateTimeColumn` and :class:`.NumberColumn`.
+    :class:`.DateTimeColumn` and :class:`.NumberColumn`.
     """
     def get_aggregate_column_type(self, column):
-        if isinstance(column, DateColumn):
-            return DateType()
-        elif isinstance(column, DateTimeColumn):
+        if isinstance(column, DateTimeColumn):
             return DateTimeType()
         elif isinstance(column, NumberColumn):
             return NumberType()
@@ -186,9 +180,7 @@ def run(self, column):
         """
         :returns: :class:`datetime.date`
         """
-        supported_columns = (DateColumn, DateTimeColumn, NumberColumn)
-
-        if not any(isinstance(column, t) for t in supported_columns):
+        if not (isinstance(column, DateTimeColumn) or isinstance(column, NumberColumn)):
             raise UnsupportedAggregationError(self, column)
 
         return max(column.get_data_without_nulls())

agate/column_types.py

@@ -41,6 +41,13 @@ class ColumnType(object): #pragma: no cover
     def __init__(self, null_values=DEFAULT_NULL_VALUES):
         self.null_values = null_values
 
+    @classmethod
+    def test(cls, d):
+        raise NotImplementedError
+
+    def cast(self, d):
+        raise NotImplementedError
+
     def _create_column(self, table, index):
         raise NotImplementedError
 
@@ -59,6 +66,27 @@ def __init__(self, true_values=DEFAULT_TRUE_VALUES, false_values=DEFAULT_FALSE_V
         self.true_values = true_values
         self.false_values = false_values
 
+    @classmethod
+    def test(cls, d):
+        """
+        Test, for purposes of type inference, if a string value could possibly
+        be valid for this column type.
+        """
+        d = d.replace(',' ,'').strip()
+
+        d_lower = d.lower()
+
+        if d_lower in DEFAULT_NULL_VALUES:
+            return True
+
+        if d_lower in DEFAULT_TRUE_VALUES:
+            return True
+
+        if d_lower in DEFAULT_FALSE_VALUES:
+            return True
+
+        return False
+
     def cast(self, d):
         """
         Cast a single value to :class:`bool`.
@@ -90,53 +118,39 @@ def _create_column(self, table, index):
 
         return BooleanColumn(table, index)
 
-class DateType(ColumnType):
+class DateTimeType(ColumnType):
     """
-    Column type for :class:`DateColumn`.
+    Column type for :class:`DateTimeColumn`.
     """
-    def __init__(self, date_format=None, null_values=DEFAULT_NULL_VALUES):
-        super(DateType, self).__init__(null_values=null_values)
+    def __init__(self, datetime_format=None, null_values=DEFAULT_NULL_VALUES):
+        super(DateTimeType, self).__init__(null_values=null_values)
 
-        self.date_format = date_format
+        self.datetime_format = datetime_format
 
-    def cast(self, d):
+    @classmethod
+    def test(cls, d):
         """
-        Cast a single value to a :class:`datetime.date`.
-
-        :param date_format: An optional :func:`datetime.strptime`
-            format string for parsing dates in this column.
-        :returns: :class:`datetime.date` or :code:`None`.
+        Test, for purposes of type inference, if a string value could possibly
+        be valid for this column type.
         """
-        if isinstance(d, datetime.date) or d is None:
-            return d
-
-        if isinstance(d, six.string_types):
-            d = d.strip()
+        d = d.strip()
 
-            if d.lower() in self.null_values:
-                return None
-
-        if self.date_format:
-            return datetime.datetime.strptime(d, self.date_format).date()
+        if d.lower() in DEFAULT_NULL_VALUES:
+            return True
 
+        # Ignore numerical values--these are never dates for inference purposes
         try:
-            return parse(d).date()
-        except (TypeError, ValueError):
-            raise CastError('Can not parse value "%s" to as datetime for DateColumn.' % d)
-
-    def _create_column(self, table, index):
-        from agate.columns import DateColumn
-
-        return DateColumn(table, index)
+            Decimal(d)
+            return False
+        except InvalidOperation:
+            pass
 
-class DateTimeType(ColumnType):
-    """
-    Column type for :class:`DateTimeColumn`.
-    """
-    def __init__(self, datetime_format=None, null_values=DEFAULT_NULL_VALUES):
-        super(DateTimeType, self).__init__(null_values=null_values)
+        try:
+            parse_result = parse(d)
+        except:
+            return False
 
-        self.datetime_format = datetime_format
+        return True
 
     def cast(self, d):
         """
@@ -160,7 +174,7 @@ def cast(self, d):
 
         try:
             return parse(d)
-        except (TypeError, ValueError):
+        except:
             raise CastError('Can not parse value "%s" to as datetime for DateTimeColumn.' % d)
 
     def _create_column(self, table, index):
@@ -172,6 +186,24 @@ class TimeDeltaType(ColumnType):
     """
     Column type for :class:`datetime.timedelta`.
     """
+    @classmethod
+    def test(self, d):
+        """
+        Test, for purposes of type inference, if a string value could possibly
+        be valid for this column type.
+        """
+        d = d.strip()
+
+        if d.lower() in DEFAULT_NULL_VALUES:
+            return True
+
+        seconds = pytimeparse.parse(d)
+
+        if seconds is None:
+            return False
+
+        return True
+
     def cast(self, d):
         """
         Cast a single value to :class:`datetime.timedelta`.
@@ -204,6 +236,23 @@ class NumberType(ColumnType):
     """
     Column type for :class:`NumberColumn`.
     """
+    @classmethod
+    def test(cls, d):
+        """
+        Test, for purposes of type inference, if a string value could possibly
+        be valid for this column type.
+        """
+        d = d.replace(',' ,'').strip()
+
+        if d.lower() in DEFAULT_NULL_VALUES:
+            return True
+
+        try:
+            Decimal(d)
+            return True
+        except InvalidOperation:
+            return False
+
     def cast(self, d):
         """
         Cast a single value to a :class:`decimal.Decimal`.
@@ -239,6 +288,10 @@ class TextType(ColumnType):
     """
     @classmethod
     def test(cls, d):
+        """
+        Test, for purposes of type inference, if a string value could possibly
+        be valid for this column type.
+        """
         return True
 
     def cast(self, d):

agate/columns/__init__.py

@@ -16,7 +16,6 @@
 
 from agate.columns.base import *
 from agate.columns.boolean import *
-from agate.columns.date import *
 from agate.columns.date_time import *
 from agate.columns.number import *
 from agate.columns.text import *

agate/computations.py

@@ -73,7 +73,7 @@ def _validate(self, table):
         before_column = table.columns[self._before_column_name]
         after_column = table.columns[self._after_column_name]
 
-        for column_type in (NumberColumn, DateColumn, DateTimeColumn, TimeDeltaColumn):
+        for column_type in (NumberColumn, DateTimeColumn, TimeDeltaColumn):
             if isinstance(before_column, column_type):
                 if not isinstance(after_column, column_type):
                     raise ValueError('Specified columns must be of the same type')
@@ -91,9 +91,7 @@ def _validate(self, table):
     def get_computed_column_type(self, table):
         before_column, after_column = self._validate(table)
 
-        if isinstance(before_column, DateColumn):
-            return TimeDeltaType()
-        elif isinstance(before_column, DateTimeColumn):
+        if isinstance(before_column, DateTimeColumn):
             return TimeDeltaType()
         elif isinstance(before_column, TimeDeltaColumn):
             return TimeDeltaType()

agate/inference.py

@@ -5,18 +5,24 @@
 from agate.column_types import *
 from agate.exceptions import *
 
-def infer_types(rows, overrides={}):
-    possible_types = {
-        'boolean': BooleanType(),
-        'date': DateType(),
-        'datetime': DateTimeType(),
-        'timedelta': TimeDeltaType(),
-        'number': NumberType(),
-        'text': TextType()
-    }
+def infer_types(rows, force={}):
+    """
+    Infer types for the columns in a given set of data.
+
+    :param force: A dictionary where each key is a column name and each value
+        is a :class:`.ColumnType` that overrides inference.
+    """
+    # In order of preference
+    possible_types = [
+        BooleanType,
+        NumberType,
+        TimeDeltaType,
+        DateTimeType,
+        TextType
+    ]
 
     num_columns = len(rows[0])
-    hypotheses = [set(possible_types.values()) for i in range(num_columns)]
+    hypotheses = [set(possible_types) for i in range(num_columns)]
 
     for row in rows:
         for i in range(num_columns):
@@ -26,30 +32,18 @@ def infer_types(rows, overrides={}):
                 continue
 
             for column_type in copy(h):
-                try:
-                    column_type.cast(row[i])
-                except CastError:
+                if not column_type.test(row[i]):
                     h.remove(column_type)
 
-    preference_order = [
-        possible_types['boolean'],
-        possible_types['number'],
-        possible_types['timedelta'],
-        possible_types['datetime'],
-        # possible_types['date'],
-        possible_types['text']
-    ]
-
     column_types = []
 
     for i in range(num_columns):
         h = hypotheses[i]
 
-        for t in preference_order:
+        # Select in prefer order
+        for t in possible_types:
             if t in h:
                 column_types.append(t)
                 break
-        else:
-            column_types.append(None)
 
     return column_types

docs/api/columns.rst

@@ -15,11 +15,6 @@ agate.columns
     :undoc-members:
     :show-inheritance:
 
-.. automodule:: agate.columns.date
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 .. automodule:: agate.columns.date_time
     :members:
     :undoc-members: