From 953e860f31a1abdd728ad769a32ca52f3bb1581d Mon Sep 17 00:00:00 2001 From: Mike McKerns Date: Fri, 28 Jun 2013 09:42:28 -0700 Subject: [PATCH 01/20] Initial commit --- README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..d6049265 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +dill +==== + +serialize all of python From eb9b240744ba7e78b3476b731ec5bf1ddca68112 Mon Sep 17 00:00:00 2001 From: Mike McKerns Date: Thu, 11 Jul 2013 10:26:28 -0700 Subject: [PATCH 02/20] merged changes to README.md from svn --- README.md | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d6049265..f1d7b1df 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,102 @@ dill ==== - serialize all of python + +About Dill +---------- +Dill extends python's 'pickle' module for serializing and de-serializing +python objects to the majority of the built-in python types. Serialization +is the process of converting an object to a byte stream, and the inverse +of which is converting a byte stream back to on python object hierarchy. + +Dill provides the user the same interface as the 'pickle' module, and +also includes some additional features. In addition to pickling python +objects, dill provides the ability to save the state of an interpreter +session in a single command. Hence, it would be feasable to save a +interpreter session, close the interpreter, ship the pickled file to +another computer, open a new interpreter, unpickle the session and +thus continue from the 'saved' state of the original interpreter +session. + +Dill can be used to store python objects to a file, but the primary +usage is to send python objects across the network as a byte stream. +Dill is quite flexible, and allows arbitrary user defined classes +and funcitons to be serialized. Thus dill is not intended to be +secure against erroneously or maliciously constructed data. It is +left to the user to decide whether the data they unpickle is from +a trustworthy source. + +Dill is part of pathos, a python framework for heterogenous computing. +Dill is in the early development stages, and any user feedback is +highly appreciated. Contact Mike McKerns [mmckerns at caltech dot edu] +with comments, suggestions, and any bugs you may find. A list of known +issues is maintained at http://trac.mystic.cacr.caltech.edu/project/pathos/query. + + +Major Features +-------------- +Dill can pickle the following standard types:: + * none, type, bool, int, long, float, complex, str, unicode, + * tuple, list, dict, file, buffer, builtin, + * both old and new style classes, + * instances of old and new style classes, + * set, frozenset, array, functions, exceptions + +Dill can also pickle more 'exotic' standard types:: + * functions with yields, nested functions, lambdas + * cell, method, unboundmethod, module, code, + * dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, + * wrapperdescriptor, xrange, slice, + * notimplemented, ellipsis, quit + +Dill cannot yet pickle these standard types:: + * frame, generator, traceback + +Dill also provides the capability to:: + * save and load python interpreter sessions + +Current Release +--------------- +The latest released version of dill is available from:: + http://trac.mystic.cacr.caltech.edu/project/pathos + +Dill is distributed under a modified BSD license. + +Development Release +------------------- +You can get the latest development release with all the shiny new features at:: + http://dev.danse.us/packages. + +or even better, fork us on our github mirror of the svn trunk:: + https://github.com/uqfoundation + +Citation +-------- +If you use dill to do research that leads to publication, we ask that you +acknowledge use of dill by citing the following in your publication:: + + M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis, + "Building a framework for predictive science", Proceedings of + the 10th Python in Science Conference, 2011; + http://arxiv.org/pdf/1202.1056 + + Michael McKerns and Michael Aivazis, + "pathos: a framework for heterogeneous computing", 2010- ; + http://trac.mystic.cacr.caltech.edu/project/pathos + +More Information +---------------- +Probably the best way to get started is to look at the tests +that are provide within dill. See `dill.tests` for a set of scripts +that test dill's ability to serialize different python objects. +Since dill conforms to the 'pickle' interface, the examples and +documentation at http://docs.python.org/library/pickle.html also +apply to dill if one will `import dill as pickle`. Dill's source code is also generally well documented, +so further questions may be resolved by inspecting the code itself, or through +browsing the reference manual. For those who like to leap before +they look, you can jump right to the installation instructions. If the aforementioned documents +do not adequately address your needs, please send us feedback. + +Dill is an active research tool. There are a growing number of publications and presentations that +discuss real-world examples and new features of dill in greater detail than presented in the user's guide. +If you would like to share how you use dill in your work, please send us a link. From 3d50bce43d2234406bdf50be6c12ea85588a9b4d Mon Sep 17 00:00:00 2001 From: Mike McKerns Date: Thu, 11 Jul 2013 11:22:37 -0700 Subject: [PATCH 03/20] fixed formatting in README.md --- README.md | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f1d7b1df..485fab99 100644 --- a/README.md +++ b/README.md @@ -36,24 +36,28 @@ issues is maintained at http://trac.mystic.cacr.caltech.edu/project/pathos/query Major Features -------------- Dill can pickle the following standard types:: - * none, type, bool, int, long, float, complex, str, unicode, - * tuple, list, dict, file, buffer, builtin, - * both old and new style classes, - * instances of old and new style classes, - * set, frozenset, array, functions, exceptions + +* none, type, bool, int, long, float, complex, str, unicode, +* tuple, list, dict, file, buffer, builtin, +* both old and new style classes, +* instances of old and new style classes, +* set, frozenset, array, functions, exceptions Dill can also pickle more 'exotic' standard types:: - * functions with yields, nested functions, lambdas - * cell, method, unboundmethod, module, code, - * dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, - * wrapperdescriptor, xrange, slice, - * notimplemented, ellipsis, quit + +* functions with yields, nested functions, lambdas +* cell, method, unboundmethod, module, code, +* dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, +* wrapperdescriptor, xrange, slice, +* notimplemented, ellipsis, quit Dill cannot yet pickle these standard types:: - * frame, generator, traceback + +* frame, generator, traceback Dill also provides the capability to:: - * save and load python interpreter sessions + +* save and load python interpreter sessions Current Release --------------- From d797b0be1ae67a18b0a4a1c865caef97ab714096 Mon Sep 17 00:00:00 2001 From: roryk Date: Wed, 23 Oct 2013 01:04:39 -0400 Subject: [PATCH 04/20] Fixed _IS_PY3 typo. --- dill/dill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dill/dill.py b/dill/dill.py index 429766a6..84d97025 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -434,7 +434,7 @@ def save_module_dict(pickler, obj): pickler.write('c__builtin__\n__main__\n') elif not is_dill(pickler) and obj is _main_module.__dict__: log.info("D3: Date: Fri, 14 Feb 2014 15:42:12 -0500 Subject: [PATCH 05/20] Set _main_module when extending StockPickler https://github.com/uqfoundation/dill/issues/23 --- dill/dill.py | 8 ++++++++ tests/test_extendpickle.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/test_extendpickle.py diff --git a/dill/dill.py b/dill/dill.py index 85c63837..494553ac 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -189,6 +189,10 @@ class Pickler(StockPickler): _byref = False pass + def __init__(self, *args, **kwargs): + StockPickler.__init__(self, *args, **kwargs) + self._main_module = _main_module + class Unpickler(StockUnpickler): """python's Unpickler extended to interpreter sessions and more types""" _main_module = None @@ -200,6 +204,10 @@ def find_class(self, module, name): return StockUnpickler.find_class(self, module, name) pass + def __init__(self, *args, **kwargs): + StockUnpickler.__init__(self, *args, **kwargs) + self._main_module = _main_module + ''' def dispatch_table(): """get the dispatch table of registered types""" diff --git a/tests/test_extendpickle.py b/tests/test_extendpickle.py new file mode 100644 index 00000000..6b37a594 --- /dev/null +++ b/tests/test_extendpickle.py @@ -0,0 +1,20 @@ +import dill as pickle +import StringIO + +def my_fn(x): + return x * 17 + +obj = lambda : my_fn(34) +assert obj() == 578 + +obj_io = StringIO.StringIO() +pickler = pickle.Pickler(obj_io) +pickler.dump(obj) + +obj_str = obj_io.getvalue() + +obj2_io = StringIO.StringIO(obj_str) +unpickler = pickle.Unpickler(obj2_io) +obj2 = unpickler.load() + +assert obj2() == 578 From a05aa499ead9bd389ce3b464e7de4295f8d2e40d Mon Sep 17 00:00:00 2001 From: Bob Fischer Date: Mon, 3 Mar 2014 08:00:33 -0500 Subject: [PATCH 06/20] making this test 3.x compatible --- tests/test_extendpickle.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_extendpickle.py b/tests/test_extendpickle.py index 6b37a594..ea0eea43 100644 --- a/tests/test_extendpickle.py +++ b/tests/test_extendpickle.py @@ -1,5 +1,8 @@ import dill as pickle -import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import BytesIO as StringIO def my_fn(x): return x * 17 From 4ccbdcbc51f447335666d9a2011040d8e74e8614 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 1 Apr 2014 16:53:03 +0100 Subject: [PATCH 07/20] Creates a safe mode for _import_module, which returns None when the module cannot be found --- dill/dill.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 24358409..d53ea241 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -404,14 +404,19 @@ def _dict_from_dictproxy(dictproxy): _dict.pop('__weakref__', None) return _dict -def _import_module(import_name): - if '.' in import_name: - items = import_name.split('.') - module = '.'.join(items[:-1]) - obj = items[-1] - else: - return __import__(import_name) - return getattr(__import__(module, None, None, [obj]), obj) +def _import_module(import_name, safe=False): + try: + if '.' in import_name: + items = import_name.split('.') + module = '.'.join(items[:-1]) + obj = items[-1] + else: + return __import__(import_name) + return getattr(__import__(module, None, None, [obj]), obj) + except ImportError: + if safe: + return None + raise def _locate_function(obj, session=False): if obj.__module__ == '__main__': # and session: From a75d42437ba5e0c3554a4b7c71ad9ad2c0482003 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 1 Apr 2014 16:56:48 +0100 Subject: [PATCH 08/20] Fixes assertion errors caused by pickling a decorator function from a module which is not being run as main. --- dill/dill.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dill/dill.py b/dill/dill.py index d53ea241..2bfe1285 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -464,6 +464,12 @@ def save_module_dict(pickler, obj): pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) else: pickler.write('c__main__\n__dict__\n') #XXX: works in general? + elif '__name__' in obj and obj != _main_module.__dict__ \ + and obj is getattr(_import_module(obj['__name__'], safe=True), '__dict__', None): + if PYTHON3: + pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8')) + else: + pickler.write('c%s\n__dict__\n' % obj['__name__']) else: log.info("D2: Date: Tue, 1 Apr 2014 17:22:48 +0100 Subject: [PATCH 09/20] Change _import_module safe default to True, and simplified _locate_function. --- dill/dill.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 2bfe1285..27a807a7 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -404,7 +404,7 @@ def _dict_from_dictproxy(dictproxy): _dict.pop('__weakref__', None) return _dict -def _import_module(import_name, safe=False): +def _import_module(import_name, safe=True): try: if '.' in import_name: items = import_name.split('.') @@ -413,7 +413,7 @@ def _import_module(import_name, safe=False): else: return __import__(import_name) return getattr(__import__(module, None, None, [obj]), obj) - except ImportError: + except (ImportError, AttributeError): if safe: return None raise @@ -421,10 +421,7 @@ def _import_module(import_name, safe=False): def _locate_function(obj, session=False): if obj.__module__ == '__main__': # and session: return False - try: - found = _import_module(obj.__module__ + '.' + obj.__name__) - except: - return False + found = _import_module(obj.__module__ + '.' + obj.__name__) return found is obj @register(CodeType) From 3b05f0ee80f92e1012c9908ea8c7ffbb856d8456 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 1 Apr 2014 17:44:51 +0100 Subject: [PATCH 10/20] Back to safe=False for _import_module --- dill/dill.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 27a807a7..070dbafc 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -404,7 +404,7 @@ def _dict_from_dictproxy(dictproxy): _dict.pop('__weakref__', None) return _dict -def _import_module(import_name, safe=True): +def _import_module(import_name, safe=False): try: if '.' in import_name: items = import_name.split('.') @@ -421,7 +421,7 @@ def _import_module(import_name, safe=True): def _locate_function(obj, session=False): if obj.__module__ == '__main__': # and session: return False - found = _import_module(obj.__module__ + '.' + obj.__name__) + found = _import_module(obj.__module__ + '.' + obj.__name__, safe=True) return found is obj @register(CodeType) From 59664ca99da105019692534fdde18d68c8da6f4a Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 1 Apr 2014 18:24:53 +0100 Subject: [PATCH 11/20] Minor change which fixes some problems with running dill under doctest --- dill/dill.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 070dbafc..919c655d 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -449,13 +449,13 @@ def save_function(pickler, obj): @register(dict) def save_module_dict(pickler, obj): - if is_dill(pickler) and obj is pickler._main_module.__dict__: + if is_dill(pickler) and obj == pickler._main_module.__dict__: log.info("D1: Date: Sat, 26 Apr 2014 16:38:05 +0100 Subject: [PATCH 12/20] Fix problem with pickling files --- dill/dill.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index ff468a90..c71e219d 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -92,11 +92,11 @@ def _trace(boolean): SuperType = type(super(Exception, TypeError())) ItemGetterType = type(itemgetter(0)) AttrGetterType = type(attrgetter('__repr__')) -FileType = open(os.devnull, 'rb', buffering=0) -TextWrapperType = open(os.devnull, 'r', buffering=-1) -BufferedRandomType = open(os.devnull, 'r+b', buffering=-1) -BufferedReaderType = open(os.devnull, 'rb', buffering=-1) -BufferedWriterType = open(os.devnull, 'wb', buffering=-1) +FileType = type(open(os.devnull, 'rb', buffering=0)) +TextWrapperType = type(open(os.devnull, 'r', buffering=-1)) +BufferedRandomType = type(open(os.devnull, 'r+b', buffering=-1)) +BufferedReaderType = type(open(os.devnull, 'rb', buffering=-1)) +BufferedWriterType = type(open(os.devnull, 'wb', buffering=-1)) try: from cStringIO import StringIO, InputType, OutputType except ImportError: From 89a2f917abcbe4d0c63cf8891b6d1f53e6a84154 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Sat, 10 May 2014 20:49:27 +0100 Subject: [PATCH 13/20] Fixes small python3 compatibility issue --- dill/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dill/source.py b/dill/source.py index 71a48f08..96b2c852 100644 --- a/dill/source.py +++ b/dill/source.py @@ -124,7 +124,7 @@ def findsource(object): else: # not a lambda, just look for the name if name in line: # need to check for decorator... hats = 0 - for _lnum in xrange(lnum-1,-1,-1): + for _lnum in range(lnum-1,-1,-1): if pat2.match(lines[_lnum]): hats += 1 else: break lnum = lnum - hats From 8e241ddf48a4a773556319760370faf3944dc4a6 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 27 May 2014 12:28:14 +0100 Subject: [PATCH 14/20] Allows saving of a modules __dict__ attribute --- dill/dill.py | 11 ++++++++--- tests/test_module.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 tests/test_module.py diff --git a/dill/dill.py b/dill/dill.py index 9a8d5c45..600691f0 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -756,11 +756,16 @@ def save_weakproxy(pickler, obj): @register(ModuleType) def save_module(pickler, obj): - if is_dill(pickler) and obj is pickler._main_module: + # if a module file name starts with this, it should be a standard module, + # so should be pickled as a reference + prefix = sys.base_prefix if PY3 else sys.prefix + if obj.__name__ not in ("builtins", "dill") \ + and not getattr(obj, "__file__", "").startswith(prefix): log.info("M1: %s" % obj) _main_dict = obj.__dict__.copy() #XXX: better no copy? option to copy? - [_main_dict.pop(item,None) for item in singletontypes] - pickler.save_reduce(__import__, (obj.__name__,), obj=obj, + [_main_dict.pop(item, None) for item in singletontypes + + ["__builtins__", "__loader__"]] + pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, state=_main_dict) else: log.info("M2: %s" % obj) diff --git a/tests/test_module.py b/tests/test_module.py new file mode 100644 index 00000000..bd5cbdcd --- /dev/null +++ b/tests/test_module.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2014 California Institute of Technology. +# License: 3-clause BSD. The full license text is available at: +# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/dill/LICENSE + +import sys +import dill +import test_mixins as module + +module.a = 1234 + +pik_mod = dill.dumps(module) + +module.a = 0 + +# remove module +del sys.modules[module.__name__] +del module + +module = dill.loads(pik_mod) +assert module.a == 1234 +assert module.double_add(1, 2, 3) == 2 * module.fx From 5e57dce84ffe7be7e699af1e2be953d5a65d8435 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 27 May 2014 15:05:50 +0100 Subject: [PATCH 15/20] Add code to clean up --- tests/test_module.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_module.py b/tests/test_module.py index bd5cbdcd..e3f2f621 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -9,6 +9,9 @@ import dill import test_mixins as module +cached = (module.__cached__ if hasattr(module, "__cached__") + else module.__file__ + "c") + module.a = 1234 pik_mod = dill.dumps(module) @@ -20,5 +23,11 @@ del module module = dill.loads(pik_mod) -assert module.a == 1234 +assert hasattr(module, "a") and module.a == 1234 assert module.double_add(1, 2, 3) == 2 * module.fx + +# clean up +import os +os.remove(cached) +if os.path.exists("__pycache__") and not os.listdir("__pycache__"): + os.removedirs("__pycache__") From 5b459b0ea230c879819056c0e8923cf0ba914353 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 27 May 2014 15:28:44 +0100 Subject: [PATCH 16/20] Fixes error with dealing with modules without a file --- dill/dill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dill/dill.py b/dill/dill.py index 600691f0..6426c3e6 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -760,7 +760,7 @@ def save_module(pickler, obj): # so should be pickled as a reference prefix = sys.base_prefix if PY3 else sys.prefix if obj.__name__ not in ("builtins", "dill") \ - and not getattr(obj, "__file__", "").startswith(prefix): + and not getattr(obj, "__file__", prefix).startswith(prefix): log.info("M1: %s" % obj) _main_dict = obj.__dict__.copy() #XXX: better no copy? option to copy? [_main_dict.pop(item, None) for item in singletontypes From 7bcf040a7aed66aab73d0c6a60e6af9715e62822 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 27 May 2014 17:45:05 +0100 Subject: [PATCH 17/20] Check for __main__ module when saving --- dill/dill.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dill/dill.py b/dill/dill.py index 6426c3e6..25ddb23c 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -759,8 +759,9 @@ def save_module(pickler, obj): # if a module file name starts with this, it should be a standard module, # so should be pickled as a reference prefix = sys.base_prefix if PY3 else sys.prefix + std_mod = getattr(obj, "__file__", prefix).startswith(prefix) if obj.__name__ not in ("builtins", "dill") \ - and not getattr(obj, "__file__", prefix).startswith(prefix): + and not std_mod or is_dill(pickler) and obj is pickler._main_module: log.info("M1: %s" % obj) _main_dict = obj.__dict__.copy() #XXX: better no copy? option to copy? [_main_dict.pop(item, None) for item in singletontypes From 7f11913ca8e2339b5fa2fbc24703ddea922f6140 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Wed, 28 May 2014 12:31:05 +0100 Subject: [PATCH 18/20] Add code to clean up when run by python 3 Also clean whitespace --- tests/test_nested.py | 157 +++++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 72 deletions(-) diff --git a/tests/test_nested.py b/tests/test_nested.py index 27789a69..eb32e909 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -9,97 +9,110 @@ """ import dill as pickle +import math #import pickle # the nested function: pickle should fail here, but dill is ok. def adder(augend): - zero = [0] - def inner(addend): - return addend+augend+zero[0] - return inner + zero = [0] + + def inner(addend): + return addend + augend + zero[0] + return inner # rewrite the nested function using a class: standard pickle should work here. class cadder(object): - def __init__(self,augend): - self.augend = augend - self.zero = [0] - def __call__(self,addend): - return addend+self.augend+self.zero[0] + def __init__(self, augend): + self.augend = augend + self.zero = [0] + + def __call__(self, addend): + return addend + self.augend + self.zero[0] # rewrite again, but as an old-style class class c2adder: - def __init__(self,augend): - self.augend = augend - self.zero = [0] - def __call__(self,addend): - return addend+self.augend+self.zero[0] + def __init__(self, augend): + self.augend = augend + self.zero = [0] + + def __call__(self, addend): + return addend + self.augend + self.zero[0] # some basic stuff -a = [0,1,2] -import math +a = [0, 1, 2] # some basic class stuff class basic(object): - pass + pass + class basic2: - pass + pass if __name__ == '__main__': - x = 5; y = 1 - - # pickled basic stuff - pa = pickle.dumps(a) - pmath = pickle.dumps(math) #XXX: FAILS in pickle - pmap = pickle.dumps(map) - # ... - la = pickle.loads(pa) - lmath = pickle.loads(pmath) - lmap = pickle.loads(pmap) - assert list(map(math.sin,a)) == list(lmap(lmath.sin,la)) - - # pickled basic class stuff - pbasic2 = pickle.dumps(basic2) - _pbasic2 = pickle.loads(pbasic2)() - pbasic = pickle.dumps(basic) - _pbasic = pickle.loads(pbasic)() - - # pickled c2adder - pc2adder = pickle.dumps(c2adder) - pc2add5 = pickle.loads(pc2adder)(x) - assert pc2add5(y) == x+y - - # pickled cadder - pcadder = pickle.dumps(cadder) - pcadd5 = pickle.loads(pcadder)(x) - assert pcadd5(y) == x+y - - # raw adder and inner - add5 = adder(x) - assert add5(y) == x+y - - # pickled adder - padder = pickle.dumps(adder) - padd5 = pickle.loads(padder)(x) - assert padd5(y) == x+y - - # pickled inner - pinner = pickle.dumps(add5) #XXX: FAILS in pickle - p5add = pickle.loads(pinner) - assert p5add(y) == x+y - - # testing moduledict where not __main__ - try: - import test_moduledict - error = None - except: - import sys - error = sys.exc_info()[1] - assert error is None - # clean up - import os - name = 'test_moduledict.py' - if os.path.exists(name) and os.path.exists(name+'c'): os.remove(name+'c') + x = 5 + y = 1 + + # pickled basic stuff + pa = pickle.dumps(a) + pmath = pickle.dumps(math) #XXX: FAILS in pickle + pmap = pickle.dumps(map) + # ... + la = pickle.loads(pa) + lmath = pickle.loads(pmath) + lmap = pickle.loads(pmap) + assert list(map(math.sin, a)) == list(lmap(lmath.sin, la)) + + # pickled basic class stuff + pbasic2 = pickle.dumps(basic2) + _pbasic2 = pickle.loads(pbasic2)() + pbasic = pickle.dumps(basic) + _pbasic = pickle.loads(pbasic)() + + # pickled c2adder + pc2adder = pickle.dumps(c2adder) + pc2add5 = pickle.loads(pc2adder)(x) + assert pc2add5(y) == x+y + + # pickled cadder + pcadder = pickle.dumps(cadder) + pcadd5 = pickle.loads(pcadder)(x) + assert pcadd5(y) == x+y + + # raw adder and inner + add5 = adder(x) + assert add5(y) == x+y + + # pickled adder + padder = pickle.dumps(adder) + padd5 = pickle.loads(padder)(x) + assert padd5(y) == x+y + + # pickled inner + pinner = pickle.dumps(add5) #XXX: FAILS in pickle + p5add = pickle.loads(pinner) + assert p5add(y) == x+y + + # testing moduledict where not __main__ + try: + import test_moduledict + error = None + except: + import sys + error = sys.exc_info()[1] + assert error is None + # clean up + import os + name = 'test_moduledict.py' + if os.path.exists(name) and os.path.exists(name+'c'): + os.remove(name+'c') + + if os.path.exists(name) and hasattr(test_moduledict, "__cached__") \ + and os.path.exists(test_moduledict.__cached__): + os.remove(getattr(test_moduledict, "__cached__")) + + if os.path.exists("__pycache__") and not os.listdir("__pycache__"): + os.removedirs("__pycache__") # EOF From 8ea9759164d2d860cb6292fa41571924b657c74a Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Wed, 28 May 2014 16:49:53 +0100 Subject: [PATCH 19/20] Fix pickling of std(in, out, err) streams --- dill/dill.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 25ddb23c..84777570 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -340,7 +340,7 @@ def _create_filehandle(name, mode, position, closed, open=open): # buffering=0 raise UnpicklingError(err) #XXX: python default is closed '' file/mode if closed: f.close() - else: f.seek(position) + elif position >= 0: f.seek(position) return f def _create_stringi(value, position, closed): @@ -551,12 +551,15 @@ def save_file(pickler, obj): if obj.closed: position = None else: - position = obj.tell() + if obj in (sys.__stdout__, sys.__stderr__, sys.__stdin__): + position = -1 + else: + position = obj.tell() pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, \ obj.closed), obj=obj) return -if PyTextWrapperType: +if PyTextWrapperType: #XXX: are stdout, stderr or stdin ever _pyio files? @register(PyBufferedRandomType) @register(PyBufferedReaderType) @register(PyBufferedWriterType) From 848e49d9a46a8c721cd15d075225bd2a10258ed4 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Wed, 4 Jun 2014 10:01:55 +0100 Subject: [PATCH 20/20] Fix pickling of partials when there are no kwargs --- dill/dill.py | 4 ++++ tests/test_functors.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/test_functors.py diff --git a/dill/dill.py b/dill/dill.py index 84777570..806f1a6a 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -310,6 +310,10 @@ def _create_function(fcode, fglobals, fname=None, fdefaults=None, \ return func def _create_ftype(ftypeobj, func, args, kwds): + if kwds is None: + kwds = {} + if args is None: + args = () return ftypeobj(func, *args, **kwds) def _create_lock(locked, *args): diff --git a/tests/test_functors.py b/tests/test_functors.py new file mode 100644 index 00000000..ac051fa7 --- /dev/null +++ b/tests/test_functors.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2014 California Institute of Technology. +# License: 3-clause BSD. The full license text is available at: +# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/dill/LICENSE + +import functools +import dill + +def f(a, b, c): # without keywords + pass + +def g(a, b, c=2): # with keywords + pass + +def h(a=1, b=2, c=3): # without args + pass + +fp = functools.partial(f, 1, 2) +gp = functools.partial(g, 1, c=2) +hp = functools.partial(h, 1, c=2) +bp = functools.partial(int, base=2) + +assert dill.pickles(fp, safe=True) +assert dill.pickles(gp, safe=True) +assert dill.pickles(hp, safe=True) +assert dill.pickles(bp, safe=True)