From 98e8913d2ac49ef9203e2f651a3d54fb166d1bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jyn=20Spring=20=E7=90=B4=E6=98=A5?= Date: Thu, 16 May 2024 11:47:33 +0200 Subject: [PATCH] BUG: Use Python pickle protocol version 4 for np.save (#26388) * BUG: Use default Python pickle protocol version rather than outdated protocol 3 * Update default pickle of np.save to 4 * Adds document for default pickle protocol 4 --- doc/release/upcoming_changes/26388.performance.rst | 3 +++ numpy/lib/_npyio_impl.py | 2 +- numpy/lib/format.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 doc/release/upcoming_changes/26388.performance.rst diff --git a/doc/release/upcoming_changes/26388.performance.rst b/doc/release/upcoming_changes/26388.performance.rst new file mode 100644 index 000000000000..885bc28c4a78 --- /dev/null +++ b/doc/release/upcoming_changes/26388.performance.rst @@ -0,0 +1,3 @@ + * `numpy.save` now uses pickle protocol version 4 for saving arrays with + object dtype, which allows for pickle objects larger than 4GB and improves + saving speed by about 5% for large arrays. \ No newline at end of file diff --git a/numpy/lib/_npyio_impl.py b/numpy/lib/_npyio_impl.py index 8986b94fd500..8940ca4463c2 100644 --- a/numpy/lib/_npyio_impl.py +++ b/numpy/lib/_npyio_impl.py @@ -520,7 +520,7 @@ def save(file, arr, allow_pickle=True, fix_imports=True): arbitrary code) and portability (pickled objects may not be loadable on different Python installations, for example if the stored objects require libraries that are not available, and not all pickled data is - compatible between Python 2 and Python 3). + compatible between different versions of Python). Default: True fix_imports : bool, optional Only useful in forcing objects in object arrays on Python 3 to be diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 87f35a7a4f60..8e14dfe4bcab 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -741,7 +741,7 @@ def write_array(fp, array, version=None, allow_pickle=True, pickle_kwargs=None): "when allow_pickle=False") if pickle_kwargs is None: pickle_kwargs = {} - pickle.dump(array, fp, protocol=3, **pickle_kwargs) + pickle.dump(array, fp, protocol=4, **pickle_kwargs) elif array.flags.f_contiguous and not array.flags.c_contiguous: if isfileobj(fp): array.T.tofile(fp)