From 842f1f1ba76c612b12c55f9f084b4505fae093fc Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 23 May 2026 19:58:21 +0200 Subject: [PATCH 1/2] perf(load): skip Variable.load dispatch for numpy data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variable.load() and Variable.load_async() always end with ``self._data = to_duck_array(self._data)`` which, for an in-memory ``numpy.ndarray``, walks the dispatch chain only to return ``self._data`` unchanged. The whole call is pure overhead in that case — the same no-op pattern that ``IndexVariable.load`` already short-circuits. Add an ``isinstance(self._data, np.ndarray)`` guard at the top of both methods. Behavior is unchanged on chunked, ExplicitlyIndexed, or non-numpy duck-array inputs. Measured on ``isel(...).load()`` of synthetic scalar-var datasets against upstream/main (best of 5, GC off): 400 scalar vars: 0.524 ms -> 0.324 ms ~1.62x 2000 scalar vars: 2.484 ms -> 1.490 ms ~1.67x Speedup scales with the number of variables (1.44x at 50 vars -> 1.67x at 2000 vars) and is flat across per-variable data size (~1.56x from size=0 to size=10,000), confirming the saving is pure dispatch overhead removal. Refs #11352. Co-authored-by: Claude --- doc/whats-new.rst | 7 +++++++ xarray/core/variable.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0425452de8d..6fa8d9086a3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -42,6 +42,13 @@ Documentation Internal Changes ~~~~~~~~~~~~~~~~ +- Skip the ``to_duck_array`` dispatch in :py:meth:`Variable.load` and + :py:meth:`Variable.load_async` when the underlying data is already a + ``numpy.ndarray``. The dispatch was a no-op for that case but added + noticeable per-variable overhead to :py:meth:`Dataset.load` / + :py:meth:`DataArray.load` on datasets with many in-memory variables + (:issue:`11352`). + .. _whats-new.2026.04.0: diff --git a/xarray/core/variable.py b/xarray/core/variable.py index b4cdf5cf6ca..f0724919cf5 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1019,6 +1019,12 @@ def load(self, **kwargs) -> Self: DataArray.load Dataset.load """ + # Fast path: an in-memory numpy array has nothing to load. The full + # to_duck_array dispatch otherwise walks is_chunked_array, the + # ExplicitlyIndexed isinstance check, and is_duck_array only to return + # self._data unchanged. + if isinstance(self._data, np.ndarray): + return self self._data = to_duck_array(self._data, **kwargs) return self @@ -1052,6 +1058,8 @@ async def load_async(self, **kwargs) -> Self: DataArray.load_async Dataset.load_async """ + if isinstance(self._data, np.ndarray): + return self self._data = await async_to_duck_array(self._data, **kwargs) return self From b279a3a44aac673bff196c0eb8b114f8216d4e55 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 26 May 2026 09:11:52 +0200 Subject: [PATCH 2/2] fix(load): preserve fast path on ndarray subclasses without chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous `isinstance(self._data, np.ndarray)` short-circuit incorrectly returned `self` (skipping the load) for ndarray subclasses with a `chunks` attribute — test fakes like DummyChunkedArray, or any third-party chunked array implementation that subclasses ndarray. Narrow to `isinstance + not hasattr("chunks")` so plain ndarrays and non-chunked subclasses (MaskedArray, np.matrix) still skip the to_duck_array dispatch, while subclasses that advertise chunks fall through to the full path. Co-authored-by: Claude --- xarray/core/variable.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index f0724919cf5..ef7ed5ebe17 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1019,11 +1019,10 @@ def load(self, **kwargs) -> Self: DataArray.load Dataset.load """ - # Fast path: an in-memory numpy array has nothing to load. The full - # to_duck_array dispatch otherwise walks is_chunked_array, the - # ExplicitlyIndexed isinstance check, and is_duck_array only to return - # self._data unchanged. - if isinstance(self._data, np.ndarray): + # Fast path: an in-memory numpy ndarray has nothing to load. Subclasses + # that advertise a `chunks` attribute (test fakes, third-party chunked + # ndarray subclasses) must still go through to_duck_array. + if isinstance(self._data, np.ndarray) and not hasattr(self._data, "chunks"): return self self._data = to_duck_array(self._data, **kwargs) return self @@ -1058,7 +1057,7 @@ async def load_async(self, **kwargs) -> Self: DataArray.load_async Dataset.load_async """ - if isinstance(self._data, np.ndarray): + if isinstance(self._data, np.ndarray) and not hasattr(self._data, "chunks"): return self self._data = await async_to_duck_array(self._data, **kwargs) return self