BUG: fixes GH3425, raising on passed invalid dtypes for datetimelike

     fixes GH2423, astyping is now checked when using datetimelike and timedeltalike
     for valid astype to dtypes

BUG: PY3 compat for timedelta64[ns] in astype

BUG: more py3 compat

PTF

Author: jreback

RELEASE.rst

@@ -56,6 +56,10 @@ pandas 0.11.1
     Note: The default value will change in 0.12 to the "no mangle" behaviour,
     If your code relies on this behaviour, explicitly specify mangle_dupe_cols=True
     in your calls.
+  - Do not allow astypes on ``datetime64[ns]`` except to ``object``, and
+    ``timedelta64[ns]`` to ``object/int`` (GH3425_)
+  - Do not allow datetimelike/timedeltalike creation except with valid types
+    (e.g. cannot pass ``datetime64[ms]``) (GH3423_)
 
 **Bug Fixes**
 
@@ -93,8 +97,9 @@ pandas 0.11.1
 .. _GH2786: https://github.com/pydata/pandas/issues/2786
 .. _GH2194: https://github.com/pydata/pandas/issues/2194
 .. _GH3230: https://github.com/pydata/pandas/issues/3230
-.. _GH3164: https://github.com/pydata/pandas/issues/3164
+.. _GH3425: https://github.com/pydata/pandas/issues/3425
 .. _GH3416: https://github.com/pydata/pandas/issues/3416
+.. _GH3423: https://github.com/pydata/pandas/issues/3423
 .. _GH3251: https://github.com/pydata/pandas/issues/3251
 .. _GH3379: https://github.com/pydata/pandas/issues/3379
 .. _GH3480: https://github.com/pydata/pandas/issues/3480

pandas/core/common.py

@@ -43,6 +43,9 @@ class AmbiguousIndexError(PandasError, KeyError):
 
 
 _POSSIBLY_CAST_DTYPES = set([ np.dtype(t) for t in ['M8[ns]','m8[ns]','O','int8','uint8','int16','uint16','int32','uint32','int64','uint64'] ])
+_NS_DTYPE = np.dtype('M8[ns]')
+_TD_DTYPE = np.dtype('m8[ns]')
+_INT64_DTYPE = np.dtype(np.int64)
 
 def isnull(obj):
     '''
@@ -1085,10 +1088,10 @@ def _possibly_cast_to_datetime(value, dtype, coerce = False):
         if is_datetime64 or is_timedelta64:
 
             # force the dtype if needed
-            #if is_datetime64 and dtype != 'datetime64[ns]':
-            #    dtype = np.dtype('datetime64[ns]')
-            #elif is_timedelta64 and dtype != 'timedelta64[ns]':
-            #    dtype = np.dtype('timedelta64[ns]')
+            if is_datetime64 and dtype != _NS_DTYPE:
+                  raise TypeError("cannot convert datetimelike to dtype [%s]" % dtype)
+            elif is_timedelta64 and dtype != _TD_DTYPE:
+                raise TypeError("cannot convert timedeltalike to dtype [%s]" % dtype)
 
             if np.isscalar(value):
                 if value == tslib.iNaT or isnull(value):
@@ -1106,7 +1109,6 @@ def _possibly_cast_to_datetime(value, dtype, coerce = False):
                         if is_datetime64:
                             from pandas.tseries.tools import to_datetime
                             value = to_datetime(value, coerce=coerce).values
-                            #value = tslib.array_to_datetime(value, coerce = coerce)
                         elif is_timedelta64:
                             value = _possibly_cast_to_timedelta(value)
                     except:
@@ -1523,9 +1525,24 @@ def _astype_nansafe(arr, dtype, copy = True):
     if not isinstance(dtype, np.dtype):
         dtype = np.dtype(dtype)
 
-    if issubclass(arr.dtype.type, np.datetime64):
+    if is_datetime64_dtype(arr):
         if dtype == object:
             return tslib.ints_to_pydatetime(arr.view(np.int64))
+        elif issubclass(dtype.type, np.int):
+            return arr.view(dtype)
+        elif dtype != _NS_DTYPE:
+            raise TypeError("cannot astype a datetimelike from [%s] to [%s]" % (arr.dtype,dtype))
+        return arr.astype(_NS_DTYPE)
+    elif is_timedelta64_dtype(arr):
+        if issubclass(dtype.type, np.int):
+            return arr.view(dtype)
+        elif dtype == object:
+            return arr.astype(object)
+
+        # in py3, timedelta64[ns] are int64
+        elif (py3compat.PY3 and dtype not in [_INT64_DTYPE,_TD_DTYPE]) or (not py3compat.PY3 and dtype != _TD_DTYPE):
+            raise TypeError("cannot astype a timedelta from [%s] to [%s]" % (arr.dtype,dtype))
+        return arr.astype(_TD_DTYPE)
     elif (np.issubdtype(arr.dtype, np.floating) and
           np.issubdtype(dtype, np.integer)):
 
@@ -1729,9 +1746,6 @@ def _check_as_is(x):
             self.queue.truncate(0)
 
 
-_NS_DTYPE = np.dtype('M8[ns]')
-
-
 def _concat_compat(to_concat, axis=0):
     # filter empty arrays
     to_concat = [x for x in to_concat if x.shape[axis] > 0]
@@ -1759,7 +1773,6 @@ def _to_pydatetime(x):
 
     return x
 
-
 def _where_compat(mask, arr1, arr2):
     if arr1.dtype == _NS_DTYPE and arr2.dtype == _NS_DTYPE:
         new_vals = np.where(mask, arr1.view(np.int64), arr2.view(np.int64))

pandas/core/internals.py

@@ -4,7 +4,7 @@
 from numpy import nan
 import numpy as np
 
-from pandas.core.common import _possibly_downcast_to_dtype, isnull
+from pandas.core.common import _possibly_downcast_to_dtype, isnull, _NS_DTYPE, _TD_DTYPE
 from pandas.core.index import Index, MultiIndex, _ensure_index, _handle_legacy_indexes
 from pandas.core.indexing import _check_slice_bounds, _maybe_convert_indices
 import pandas.core.common as com
@@ -740,10 +740,6 @@ def should_store(self, value):
                               (np.integer, np.floating, np.complexfloating,
                                np.datetime64, np.bool_))
 
-_NS_DTYPE = np.dtype('M8[ns]')
-_TD_DTYPE = np.dtype('m8[ns]')
-
-
 class DatetimeBlock(Block):
     _can_hold_na = True
 

pandas/tests/test_series.py

@@ -482,13 +482,13 @@ def test_constructor_dtype_datetime64(self):
         s.ix[0] = np.nan
         self.assert_(s.dtype == 'M8[ns]')
 
-        # GH3414 related
-        #import pdb; pdb.set_trace()
-        #result = Series(Series(dates).astype('int')/1e6,dtype='M8[ms]')
-        #self.assert_(result.dtype == 'M8[ns]')
+        # invalid astypes
+        for t in ['s','D','us','ms']:
+            self.assertRaises(TypeError, s.astype, 'M8[%s]' % t)
 
-        #s = Series(dates, dtype='datetime64')
-        #self.assert_(s.dtype == 'M8[ns]')
+        # GH3414 related
+        self.assertRaises(TypeError, lambda x: Series(Series(dates).astype('int')/1000000,dtype='M8[ms]'))
+        self.assertRaises(TypeError, lambda x: Series(dates, dtype='datetime64'))
 
     def test_constructor_dict(self):
         d = {'a': 0., 'b': 1., 'c': 2.}
@@ -1830,6 +1830,13 @@ def test_constructor_dtype_timedelta64(self):
         td = Series([ timedelta(days=i) for i in range(3) ] + [ np.nan ], dtype='m8[ns]' )
         self.assert_(td.dtype=='timedelta64[ns]')
 
+        # invalid astypes
+        for t in ['s','D','us','ms']:
+            self.assertRaises(TypeError, td.astype, 'm8[%s]' % t)
+
+        # valid astype
+        td.astype('int')
+
         # this is an invalid casting
         self.assertRaises(Exception, Series, [ timedelta(days=i) for i in range(3) ] + [ 'foo' ], dtype='m8[ns]' )
 

pandas/tseries/index.py

@@ -6,7 +6,7 @@
 
 import numpy as np
 
-from pandas.core.common import isnull
+from pandas.core.common import isnull, _NS_DTYPE, _INT64_DTYPE
 from pandas.core.index import Index, Int64Index
 from pandas.tseries.frequencies import (
     infer_freq, to_offset, get_period_alias,
@@ -92,9 +92,6 @@ class TimeSeriesError(Exception):
 
 
 _midnight = time(0, 0)
-_NS_DTYPE = np.dtype('M8[ns]')
-_INT64_DTYPE = np.dtype(np.int64)
-
 
 class DatetimeIndex(Int64Index):
     """

pandas/tseries/period.py

@@ -12,7 +12,7 @@
 import pandas.tseries.frequencies as _freq_mod
 
 import pandas.core.common as com
-from pandas.core.common import isnull
+from pandas.core.common import isnull, _NS_DTYPE, _INT64_DTYPE
 from pandas.util import py3compat
 
 from pandas.lib import Timestamp
@@ -516,10 +516,6 @@ def wrapper(self, other):
         return result
     return wrapper
 
-_INT64_DTYPE = np.dtype(np.int64)
-_NS_DTYPE = np.dtype('M8[ns]')
-
-
 class PeriodIndex(Int64Index):
     """
     Immutable ndarray holding ordinal values indicating regular periods in