Incidental implementation specific types (#472)

* Incidental implementation specific types

* Add incedental types to objects.py

* Remove types for objects that are automatically by Python

* Rename `SymtableStentryType` to `SymtableEntryType`

dill/_dill.py

@@ -42,6 +42,7 @@ def _trace(boolean):
 OLDER = (PY3 and sys.hexversion < 0x3040000) or (sys.hexversion < 0x2070ab1)
 OLD33 = (sys.hexversion < 0x3030000)
 OLD37 = (sys.hexversion < 0x3070000)
+OLD38 = (sys.hexversion < 0x3080000)
 OLD39 = (sys.hexversion < 0x3090000)
 OLD310 = (sys.hexversion < 0x30a0000)
 PY34 = (0x3040000 <= sys.hexversion < 0x3050000)
@@ -87,6 +88,7 @@ def _trace(boolean):
 import gc
 # import zlib
 from weakref import ReferenceType, ProxyType, CallableProxyType
+from collections import OrderedDict
 from functools import partial
 from operator import itemgetter, attrgetter
 # new in python3.3
@@ -274,8 +276,6 @@ def get_file_type(*args, **kwargs):
     except NameError: ExitType = None
     singletontypes = []
 
-from collections import OrderedDict
-
 import inspect
 
 ### Shims for different versions of Python and dill
@@ -572,7 +572,6 @@ def __init__(self, *args, **kwds):
         self._strictio = False #_strictio
         self._fmode = settings['fmode'] if _fmode is None else _fmode
         self._recurse = settings['recurse'] if _recurse is None else _recurse
-        from collections import OrderedDict
         self._postproc = OrderedDict()
 
     def dump(self, obj): #NOTE: if settings change, need to update attributes
@@ -723,6 +722,15 @@ def _create_typemap():
     'SuperType': SuperType,
     'ItemGetterType': ItemGetterType,
     'AttrGetterType': AttrGetterType,
+})
+
+# "Incidental" implementation specific types. Unpickling these types in another
+# implementation of Python (PyPy -> CPython) is not gauranteed to work
+
+# This dictionary should contain all types that appear in Python implementations
+# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types
+x=OrderedDict()
+_incedental_reverse_typemap = {
     'FileType': FileType,
     'BufferedRandomType': BufferedRandomType,
     'BufferedReaderType': BufferedReaderType,
@@ -732,18 +740,62 @@ def _create_typemap():
     'PyBufferedReaderType': PyBufferedReaderType,
     'PyBufferedWriterType': PyBufferedWriterType,
     'PyTextWrapperType': PyTextWrapperType,
-})
+}
+
+if PY3:
+    _incedental_reverse_typemap.update({
+        "DictKeysType": type({}.keys()),
+        "DictValuesType": type({}.values()),
+        "DictItemsType": type({}.items()),
+
+        "OdictKeysType": type(x.keys()),
+        "OdictValuesType": type(x.values()),
+        "OdictItemsType": type(x.items()),
+    })
+else:
+    _incedental_reverse_typemap.update({
+        "DictKeysType": type({}.viewkeys()),
+        "DictValuesType": type({}.viewvalues()),
+        "DictItemsType": type({}.viewitems()),
+    })
+
 if ExitType:
-    _reverse_typemap['ExitType'] = ExitType
+    _incedental_reverse_typemap['ExitType'] = ExitType
 if InputType:
-    _reverse_typemap['InputType'] = InputType
-    _reverse_typemap['OutputType'] = OutputType
+    _incedental_reverse_typemap['InputType'] = InputType
+    _incedental_reverse_typemap['OutputType'] = OutputType
 if not IS_PYPY:
-    _reverse_typemap['WrapperDescriptorType'] = WrapperDescriptorType
-    _reverse_typemap['MethodDescriptorType'] = MethodDescriptorType
-    _reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType
+    _incedental_reverse_typemap['WrapperDescriptorType'] = WrapperDescriptorType
+    _incedental_reverse_typemap['MethodDescriptorType'] = MethodDescriptorType
+    _incedental_reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType
 else:
-    _reverse_typemap['MemberDescriptorType'] = MemberDescriptorType
+    _incedental_reverse_typemap['MemberDescriptorType'] = MemberDescriptorType
+
+try:
+    import symtable
+    _incedental_reverse_typemap["SymtableStentryType"] = type(symtable.symtable("", "string", "exec")._table)
+except:
+    pass
+
+if sys.hexversion >= 0x30a00a0:
+    _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines())
+
+if sys.hexversion >= 0x30b00b0:
+    from types import GenericAlias
+    _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,))))
+    _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions())
+
+try:
+    import winreg
+    _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType
+except:
+    pass
+
+_reverse_typemap.update(_incedental_reverse_typemap)
+_incedental_types = set(_incedental_reverse_typemap.values())
+
+del x
+
 if PY3:
     _typemap = dict((v, k) for k, v in _reverse_typemap.items())
 else:
@@ -1988,6 +2040,8 @@ def save_module(pickler, obj):
 def save_type(pickler, obj, postproc_list=None):
     if obj in _typemap:
         log.info("T1: %s" % obj)
+        # if obj in _incedental_types:
+        #     warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning)
         pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj)
         log.info("# T1")
     elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]):
@@ -2255,6 +2309,9 @@ def save_capsule(pickler, obj):
         destructor = _PyCapsule_GetDestructor(obj)
         pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj)
         log.info("# Cap")
+    _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType
+    _reverse_typemap['PyCapsuleType'] = PyCapsuleType
+    _incedental_types.add(PyCapsuleType)
 else:
     _testcapsule = None
 

dill/_objects.py

@@ -417,9 +417,49 @@ class _Struct(ctypes.Structure):
 a['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+'
 # FIXME: FileType fails >= 3.1
 # built-in functions (CH 2)
+# Iterators:
 a['ListIteratorType'] = iter(_list) # empty vs non-empty FIXME: fail < 3.2
+x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty
 a['TupleIteratorType']= iter(_tuple) # empty vs non-empty FIXME: fail < 3.2
 a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty FIXME: fail < 3.2
+a["BytesIteratorType"] = iter(b'')
+a["BytearrayIteratorType"] = iter(bytearray(b''))
+a["CallableIteratorType"] = iter(iter, None)
+a["MemoryIteratorType"] = iter(memoryview(b''))
+a["ListReverseiteratorType"] = reversed([])
+X = a['OrderedDictType']
+a["OdictKeysType"] = X.keys()
+a["OdictValuesType"] = X.values()
+a["OdictItemsType"] = X.items()
+a["OdictIteratorType"] = iter(X.keys())
+del X
+if PY3:
+    x['DictionaryItemIteratorType'] = iter(type.__dict__.items())
+    x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys())
+    x['DictionaryValueIteratorType'] = iter(type.__dict__.values())
+else:
+    x['DictionaryItemIteratorType'] = type.__dict__.iteritems()
+    x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys()
+    x['DictionaryValueIteratorType'] = type.__dict__.itervalues()
+if sys.hexversion >= 0x30800a0:
+    a["DictReversekeyiteratorType"] = reversed({}.keys())
+    a["DictReversevalueiteratorType"] = reversed({}.values())
+    a["DictReverseitemiteratorType"] = reversed({}.items())
+
+try:
+    import symtable
+    a["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table
+except:
+    pass
+
+if sys.hexversion >= 0x30a00a0:
+    a['LineIteratorType'] = compile('3', '', 'eval').co_lines()
+
+if sys.hexversion >= 0x30b00b0:
+    from types import GenericAlias
+    a["GenericAliasIteratorType"] = iter(GenericAlias(list, (int,)))
+    a['PositionsIteratorType'] = compile('3', '', 'eval').co_positions()
+
 # data types (CH 8)
 a['PrettyPrinterType'] = pprint.PrettyPrinter() #FIXME: fail >= 3.2 and == 2.5
 # numeric and mathematical types (CH 9)
@@ -452,16 +492,7 @@ class _Struct(ctypes.Structure):
 # other (concrete) object types
 # (also: Capsule / CObject ?)
 # built-in functions (CH 2)
-x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty
 # built-in types (CH 5)
-if PY3:
-    x['DictionaryItemIteratorType'] = iter(type.__dict__.items())
-    x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys())
-    x['DictionaryValueIteratorType'] = iter(type.__dict__.values())
-else:
-    x['DictionaryItemIteratorType'] = type.__dict__.iteritems()
-    x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys()
-    x['DictionaryValueIteratorType'] = type.__dict__.itervalues()
 # string services (CH 7)
 x['StructType'] = struct.Struct('c')
 x['CallableIteratorType'] = _srepattern.finditer('')