From ced84211105a348de07fe612be4babb77d1eb187 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 8 Sep 2021 23:57:19 -0700 Subject: [PATCH 1/8] Use `tuple` for `writeable` --- distributed/protocol/tests/test_pickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index a25d499ea0..c2a293d650 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -76,7 +76,7 @@ def test_pickle_empty(): np = pytest.importorskip("numpy") x = np.arange(2)[0:0] # Empty view header, frames = pickle_dumps(x) - header["writeable"] = [False] * len(frames) + header["writeable"] = (False,) * len(frames) y = deserialize(header, frames) assert memoryview(y).nbytes == 0 assert memoryview(y).readonly From 20fbce239a382a2b96988712850075470510313a Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 8 Sep 2021 23:57:20 -0700 Subject: [PATCH 2/8] Use `serialize` instead of `pickle_dumps` --- distributed/protocol/tests/test_pickle.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index c2a293d650..f31329aabd 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -8,7 +8,6 @@ from distributed.protocol import deserialize, serialize from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads -from distributed.protocol.serialize import pickle_dumps if sys.version_info < (3, 8): try: @@ -75,7 +74,7 @@ def __reduce_ex__(self, protocol): def test_pickle_empty(): np = pytest.importorskip("numpy") x = np.arange(2)[0:0] # Empty view - header, frames = pickle_dumps(x) + header, frames = serialize(x, serializers=("pickle",)) header["writeable"] = (False,) * len(frames) y = deserialize(header, frames) assert memoryview(y).nbytes == 0 From 5cb557bf6ac55851c0df38b789049f8ae03f286b Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 8 Sep 2021 23:57:20 -0700 Subject: [PATCH 3/8] Assert pickling did occur --- distributed/protocol/tests/test_pickle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index f31329aabd..f71345fbfb 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -75,6 +75,7 @@ def test_pickle_empty(): np = pytest.importorskip("numpy") x = np.arange(2)[0:0] # Empty view header, frames = serialize(x, serializers=("pickle",)) + assert header["serializer"] == "pickle" header["writeable"] = (False,) * len(frames) y = deserialize(header, frames) assert memoryview(y).nbytes == 0 From aa757041bc8576342c9de455db476b66082cf7c2 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 8 Sep 2021 23:57:21 -0700 Subject: [PATCH 4/8] Refactor out `MemoryviewHolder` to use elsewhere --- distributed/protocol/tests/test_pickle.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index f71345fbfb..9f7a7d63a0 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -18,6 +18,17 @@ import pickle +class MemoryviewHolder: + def __init__(self, mv): + self.mv = memoryview(mv) + + def __reduce_ex__(self, protocol): + if protocol >= 5: + return MemoryviewHolder, (pickle.PickleBuffer(self.mv),) + else: + return MemoryviewHolder, (self.mv.tobytes(),) + + def test_pickle_data(): data = [1, b"123", "123", [123], {}, set()] for d in data: @@ -26,16 +37,6 @@ def test_pickle_data(): def test_pickle_out_of_band(): - class MemoryviewHolder: - def __init__(self, mv): - self.mv = memoryview(mv) - - def __reduce_ex__(self, protocol): - if protocol >= 5: - return MemoryviewHolder, (pickle.PickleBuffer(self.mv),) - else: - return MemoryviewHolder, (self.mv.tobytes(),) - mv = memoryview(b"123") mvh = MemoryviewHolder(mv) From 3a0f22a064bb809139d1e8b038070616688e56fe Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 8 Sep 2021 23:57:22 -0700 Subject: [PATCH 5/8] Use `MemoryviewHolder` instead of NumPy --- distributed/protocol/tests/test_pickle.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index 9f7a7d63a0..7aeb71081d 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -73,14 +73,13 @@ def test_pickle_out_of_band(): def test_pickle_empty(): - np = pytest.importorskip("numpy") - x = np.arange(2)[0:0] # Empty view + x = MemoryviewHolder(bytearray()) # Empty view header, frames = serialize(x, serializers=("pickle",)) assert header["serializer"] == "pickle" header["writeable"] = (False,) * len(frames) y = deserialize(header, frames) - assert memoryview(y).nbytes == 0 - assert memoryview(y).readonly + assert y.mv.nbytes == 0 + assert y.mv.readonly def test_pickle_numpy(): From 08542a0688f06c3f336a51df30ce4e01d6e9851e Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 8 Sep 2021 23:57:23 -0700 Subject: [PATCH 6/8] Check a few more things with `MemoryviewHolder` --- distributed/protocol/tests/test_pickle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index 7aeb71081d..a29091ce98 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -78,6 +78,9 @@ def test_pickle_empty(): assert header["serializer"] == "pickle" header["writeable"] = (False,) * len(frames) y = deserialize(header, frames) + assert isinstance(y, MemoryviewHolder) + assert isinstance(y.mv, memoryview) + assert y.mv == x.mv assert y.mv.nbytes == 0 assert y.mv.readonly From 0ae7863232c23849c2fd5b87d6d3fd128182260f Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 8 Sep 2021 23:57:24 -0700 Subject: [PATCH 7/8] Perform a few more baseline serialization checks --- distributed/protocol/tests/test_pickle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index a29091ce98..ad5f141bc4 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -76,6 +76,8 @@ def test_pickle_empty(): x = MemoryviewHolder(bytearray()) # Empty view header, frames = serialize(x, serializers=("pickle",)) assert header["serializer"] == "pickle" + assert len(frames) >= 1 + assert isinstance(frames[0], bytes) header["writeable"] = (False,) * len(frames) y = deserialize(header, frames) assert isinstance(y, MemoryviewHolder) From 881d64394f47d1905ff4eef94f811f39eff15ddb Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 9 Sep 2021 00:13:54 -0700 Subject: [PATCH 8/8] Handle pickle protocol 5 separately --- distributed/protocol/tests/test_pickle.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index ad5f141bc4..a4ab843564 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -75,11 +75,22 @@ def test_pickle_out_of_band(): def test_pickle_empty(): x = MemoryviewHolder(bytearray()) # Empty view header, frames = serialize(x, serializers=("pickle",)) + assert header["serializer"] == "pickle" assert len(frames) >= 1 assert isinstance(frames[0], bytes) - header["writeable"] = (False,) * len(frames) + + if HIGHEST_PROTOCOL >= 5: + assert len(frames) == 2 + assert len(header["writeable"]) == 1 + + header["writeable"] = (False,) * len(frames) + else: + assert len(frames) == 1 + assert len(header["writeable"]) == 0 + y = deserialize(header, frames) + assert isinstance(y, MemoryviewHolder) assert isinstance(y.mv, memoryview) assert y.mv == x.mv