From a2a5353f348c3c42ccf7b560a8c8dee42e21f035 Mon Sep 17 00:00:00 2001 From: Robert Bradshaw Date: Thu, 25 May 2017 13:49:48 -0700 Subject: [PATCH 1/3] [BEAM-2365] Use the highest pickle protocol available. --- sdks/python/apache_beam/coders/coders.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index ce914ddec0e0..9b346d8cd46d 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -365,7 +365,7 @@ def maybe_dill_dumps(o): # We need to use the dill pickler for objects of certain custom classes, # including, for example, ones that contain lambdas. try: - return pickle.dumps(o) + return pickle.dumps(o, -1) except Exception: # pylint: disable=broad-except return dill.dumps(o) @@ -426,7 +426,8 @@ class PickleCoder(_PickleCoderBase): """Coder using Python's pickle functionality.""" def _create_impl(self): - return coder_impl.CallbackCoderImpl(pickle.dumps, pickle.loads) + dumps = pickle.dumps + return coder_impl.CallbackCoderImpl(lambda x: dumps(x, -1), pickle.loads) class DillCoder(_PickleCoderBase): From accd42be6e64520fc94116533e7f63cef7f11289 Mon Sep 17 00:00:00 2001 From: Robert Bradshaw Date: Fri, 26 May 2017 10:49:07 -0700 Subject: [PATCH 2/3] Fix base64 pickle coder expectations. --- sdks/python/apache_beam/coders/coders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 9b346d8cd46d..8747ba43590e 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -516,7 +516,7 @@ class Base64PickleCoder(Coder): # than via a special Coder. def encode(self, value): - return base64.b64encode(pickle.dumps(value)) + return base64.b64encode(pickle.dumps(value, -1)) def decode(self, encoded): return pickle.loads(base64.b64decode(encoded)) From 45fb55a5510c7c1f4d6fd62a13bfefccdbb426b5 Mon Sep 17 00:00:00 2001 From: Robert Bradshaw Date: Fri, 26 May 2017 14:39:44 -0700 Subject: [PATCH 3/3] fixup: use HIGHEST_PROTOCOL --- sdks/python/apache_beam/coders/coders.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 8747ba43590e..f40045d142ff 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -365,7 +365,7 @@ def maybe_dill_dumps(o): # We need to use the dill pickler for objects of certain custom classes, # including, for example, ones that contain lambdas. try: - return pickle.dumps(o, -1) + return pickle.dumps(o, pickle.HIGHEST_PROTOCOL) except Exception: # pylint: disable=broad-except return dill.dumps(o) @@ -427,7 +427,9 @@ class PickleCoder(_PickleCoderBase): def _create_impl(self): dumps = pickle.dumps - return coder_impl.CallbackCoderImpl(lambda x: dumps(x, -1), pickle.loads) + HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL + return coder_impl.CallbackCoderImpl( + lambda x: dumps(x, HIGHEST_PROTOCOL), pickle.loads) class DillCoder(_PickleCoderBase): @@ -516,7 +518,7 @@ class Base64PickleCoder(Coder): # than via a special Coder. def encode(self, value): - return base64.b64encode(pickle.dumps(value, -1)) + return base64.b64encode(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)) def decode(self, encoded): return pickle.loads(base64.b64decode(encoded))