From 3893f99363b89fd091bda2cf1625bd0544c2b5c2 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Mon, 20 Oct 2025 14:49:46 +0100 Subject: [PATCH 1/8] enable callbacks for loading netcdf with multiple variables --- lib/iris/fileformats/netcdf/loader.py | 22 +++++++++++++++++++ ...__translate_constraints_to_var_callback.py | 11 +++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/iris/fileformats/netcdf/loader.py b/lib/iris/fileformats/netcdf/loader.py index 216df67590..32c5f15abf 100644 --- a/lib/iris/fileformats/netcdf/loader.py +++ b/lib/iris/fileformats/netcdf/loader.py @@ -625,6 +625,28 @@ def inner(cf_datavar): return match result = inner + elif len(constraints) > 1: + if all(isinstance(constraint, iris._constraints.NameConstraint) and constraint.STASH == "none" for constraint in constraints): + def inner(cf_datavar): + match = False + for constraint in constraints: + match = True + for name in constraint._names: + expected = getattr(constraint, name) + if name != "STASH" and expected != "none": + attr_name = "cf_name" if name == "var_name" else name + # Fetch property : N.B. CFVariable caches the property values + # The use of a default here is the only difference from the code in NameConstraint. + if not hasattr(cf_datavar, attr_name): + continue + actual = getattr(cf_datavar, attr_name, "") + if actual != expected: + match = False + break + if match: + break + return match + result = inner return result diff --git a/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py b/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py index 64c2c82007..5b3f0a0564 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py +++ b/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py @@ -33,7 +33,16 @@ class Test(tests.IrisTest): def test_multiple_constraints(self): constrs = [ iris.NameConstraint(standard_name="x_wind"), - iris.NameConstraint(var_name="var1"), + iris.NameConstraint(var_name="var2"), + ] + callback = _translate_constraints_to_var_callback(constrs) + result = [callback(var) for var in self.data_variables] + self.assertArrayEqual(result, [True, True, False, True, False]) + + def test_multiple_constraints_invalid(self): + constrs = [ + iris.NameConstraint(standard_name="x_wind"), + iris.NameConstraint(var_name="var1", STASH="m01s00i024"), ] result = _translate_constraints_to_var_callback(constrs) self.assertIsNone(result) From 795881158e67ed292889380ca4992a4b18186767 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Mon, 20 Oct 2025 14:55:40 +0100 Subject: [PATCH 2/8] ruff format --- lib/iris/fileformats/netcdf/loader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/iris/fileformats/netcdf/loader.py b/lib/iris/fileformats/netcdf/loader.py index 32c5f15abf..d57d48cd8c 100644 --- a/lib/iris/fileformats/netcdf/loader.py +++ b/lib/iris/fileformats/netcdf/loader.py @@ -626,7 +626,12 @@ def inner(cf_datavar): result = inner elif len(constraints) > 1: - if all(isinstance(constraint, iris._constraints.NameConstraint) and constraint.STASH == "none" for constraint in constraints): + if all( + isinstance(constraint, iris._constraints.NameConstraint) + and constraint.STASH == "none" + for constraint in constraints + ): + def inner(cf_datavar): match = False for constraint in constraints: @@ -646,6 +651,7 @@ def inner(cf_datavar): if match: break return match + result = inner return result From 7b22cf967896e0efd1b2d929cebe71e9b4e214f1 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Wed, 22 Oct 2025 09:21:06 +0100 Subject: [PATCH 3/8] update docstring --- lib/iris/fileformats/netcdf/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/fileformats/netcdf/loader.py b/lib/iris/fileformats/netcdf/loader.py index d57d48cd8c..327a7db5f4 100644 --- a/lib/iris/fileformats/netcdf/loader.py +++ b/lib/iris/fileformats/netcdf/loader.py @@ -592,7 +592,7 @@ def _translate_constraints_to_var_callback(constraints): Notes ----- - For now, ONLY handles a single NameConstraint with no 'STASH' component. + For now, ONLY handles NameConstraints with no 'STASH' component. """ import iris._constraints From b4d10e3969c92650bceb4eb31ec3e63c64b87b22 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Thu, 23 Oct 2025 14:48:43 +0100 Subject: [PATCH 4/8] add integration test --- lib/iris/tests/integration/netcdf/test_general.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/iris/tests/integration/netcdf/test_general.py b/lib/iris/tests/integration/netcdf/test_general.py index 40334463e5..b7f2d64cd0 100644 --- a/lib/iris/tests/integration/netcdf/test_general.py +++ b/lib/iris/tests/integration/netcdf/test_general.py @@ -314,6 +314,13 @@ def test_netcdf_with_NameConstraint(self): self.assertEqual(len(cubes), 1) self.assertEqual(cubes[0].var_name, "cdf_temp_dmax_tmean_abs") + def test_netcdf_with_2_NameConstraints(self): + var_names = ["cdf_temp_dmax_tmean_abs", "temp_dmax_tmean_abs"] + constrs = [iris.NameConstraint(var_name=var_name) for var_name in var_names] + cubes = iris.load(self.filename, constrs) + self.assertEqual(len(cubes), 2) + self.assertEqual([cube.var_name for cube in cubes], var_names) + def test_netcdf_with_no_constraint(self): cubes = iris.load(self.filename) self.assertEqual(len(cubes), 3) From 0778ae0131a26186e34cfde936a1ffd59955fe21 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Thu, 23 Oct 2025 15:31:26 +0100 Subject: [PATCH 5/8] address review comments --- lib/iris/fileformats/netcdf/loader.py | 58 ++++++------------- ...__translate_constraints_to_var_callback.py | 20 +++++++ 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/lib/iris/fileformats/netcdf/loader.py b/lib/iris/fileformats/netcdf/loader.py index 327a7db5f4..00d135ad13 100644 --- a/lib/iris/fileformats/netcdf/loader.py +++ b/lib/iris/fileformats/netcdf/loader.py @@ -598,18 +598,19 @@ def _translate_constraints_to_var_callback(constraints): import iris._constraints constraints = iris._constraints.list_of_constraints(constraints) - result = None - if len(constraints) == 1: - (constraint,) = constraints - if ( + if len(constraints) == 0 or not all( isinstance(constraint, iris._constraints.NameConstraint) and constraint.STASH == "none" - ): - # As long as it doesn't use a STASH match, then we can treat it as - # a testing against name properties of cf_var. - # That's just like testing against name properties of a cube, except that they may not all exist. - def inner(cf_datavar): - match = True + for constraint in constraints + ): + # We can define a var-filtering function to speedup the load, *ONLY* when we + # have some constraints, and all are simple NameConstraints with no STASH. + result = None + else: + def inner(cf_datavar): + match_any_constraint = False + for constraint in constraints: + match_this_constraint = True for name in constraint._names: expected = getattr(constraint, name) if name != "STASH" and expected != "none": @@ -620,39 +621,14 @@ def inner(cf_datavar): continue actual = getattr(cf_datavar, attr_name, "") if actual != expected: - match = False + match_this_constraint = False break - return match + if match_this_constraint: + match_any_constraint = True + break + return match_any_constraint - result = inner - elif len(constraints) > 1: - if all( - isinstance(constraint, iris._constraints.NameConstraint) - and constraint.STASH == "none" - for constraint in constraints - ): - - def inner(cf_datavar): - match = False - for constraint in constraints: - match = True - for name in constraint._names: - expected = getattr(constraint, name) - if name != "STASH" and expected != "none": - attr_name = "cf_name" if name == "var_name" else name - # Fetch property : N.B. CFVariable caches the property values - # The use of a default here is the only difference from the code in NameConstraint. - if not hasattr(cf_datavar, attr_name): - continue - actual = getattr(cf_datavar, attr_name, "") - if actual != expected: - match = False - break - if match: - break - return match - - result = inner + result = inner return result diff --git a/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py b/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py index 5b3f0a0564..9d5125e068 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py +++ b/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py @@ -47,6 +47,21 @@ def test_multiple_constraints_invalid(self): result = _translate_constraints_to_var_callback(constrs) self.assertIsNone(result) + def test_multiple_constraints__multiname(self): + # Modify the first constraint to require BOTH var-name and std-name match + constrs = [ + iris.NameConstraint(standard_name="x_wind", var_name="var1"), + iris.NameConstraint(var_name="var2"), + ] + callback = _translate_constraints_to_var_callback(constrs) + # Add 2 extra vars: one passes both name checks, and the other does not + vars = self.data_variables +[ + CFDataVariable("var1", MagicMock(standard_name="x_wind")), + CFDataVariable("var1", MagicMock(standard_name="air_pressure")) + ] + result = [callback(var) for var in vars] + self.assertArrayEqual(result, [True, True, False, True, False, True, False]) + def test_non_NameConstraint(self): constr = iris.AttributeConstraint(STASH="m01s00i002") result = _translate_constraints_to_var_callback(constr) @@ -100,6 +115,11 @@ def test_NameConstraint_with_STASH(self): result = _translate_constraints_to_var_callback(constr) self.assertIsNone(result) + def test_no_constraints(self): + constrs = [] + result = _translate_constraints_to_var_callback(constrs) + self.assertIsNone(result) + if __name__ == "__main__": tests.main() From 0202f8b0dc56e25c2b5c6b0317a2ed15777bb4c5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:32:12 +0000 Subject: [PATCH 6/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- lib/iris/fileformats/netcdf/loader.py | 7 ++++--- .../loader/test__translate_constraints_to_var_callback.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/iris/fileformats/netcdf/loader.py b/lib/iris/fileformats/netcdf/loader.py index 00d135ad13..f5e947c0f0 100644 --- a/lib/iris/fileformats/netcdf/loader.py +++ b/lib/iris/fileformats/netcdf/loader.py @@ -599,14 +599,15 @@ def _translate_constraints_to_var_callback(constraints): constraints = iris._constraints.list_of_constraints(constraints) if len(constraints) == 0 or not all( - isinstance(constraint, iris._constraints.NameConstraint) - and constraint.STASH == "none" - for constraint in constraints + isinstance(constraint, iris._constraints.NameConstraint) + and constraint.STASH == "none" + for constraint in constraints ): # We can define a var-filtering function to speedup the load, *ONLY* when we # have some constraints, and all are simple NameConstraints with no STASH. result = None else: + def inner(cf_datavar): match_any_constraint = False for constraint in constraints: diff --git a/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py b/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py index 9d5125e068..b95bbd0552 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py +++ b/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py @@ -55,9 +55,9 @@ def test_multiple_constraints__multiname(self): ] callback = _translate_constraints_to_var_callback(constrs) # Add 2 extra vars: one passes both name checks, and the other does not - vars = self.data_variables +[ + vars = self.data_variables + [ CFDataVariable("var1", MagicMock(standard_name="x_wind")), - CFDataVariable("var1", MagicMock(standard_name="air_pressure")) + CFDataVariable("var1", MagicMock(standard_name="air_pressure")), ] result = [callback(var) for var in vars] self.assertArrayEqual(result, [True, True, False, True, False, True, False]) From 3c7fb7c1de07984df93869b7bc9d01e91c2bca80 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Thu, 23 Oct 2025 17:04:22 +0100 Subject: [PATCH 7/8] address review comment --- lib/iris/tests/integration/netcdf/test_general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/tests/integration/netcdf/test_general.py b/lib/iris/tests/integration/netcdf/test_general.py index b7f2d64cd0..21afb6bc8b 100644 --- a/lib/iris/tests/integration/netcdf/test_general.py +++ b/lib/iris/tests/integration/netcdf/test_general.py @@ -319,7 +319,7 @@ def test_netcdf_with_2_NameConstraints(self): constrs = [iris.NameConstraint(var_name=var_name) for var_name in var_names] cubes = iris.load(self.filename, constrs) self.assertEqual(len(cubes), 2) - self.assertEqual([cube.var_name for cube in cubes], var_names) + self.assertEqual(sorted([cube.var_name for cube in cubes]), var_names) def test_netcdf_with_no_constraint(self): cubes = iris.load(self.filename) From 98a2f6ef5b63491ba18f3f71a7f33ae244dbb977 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Fri, 24 Oct 2025 09:22:31 +0100 Subject: [PATCH 8/8] add whatsnew --- docs/src/whatsnew/latest.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index dd91aa7cc8..eb4c542fe1 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -82,6 +82,11 @@ This document explains the changes made to Iris for this release is hoped that a future ``libnetcdf`` release will recover the original performance. See `netcdf-c#3183`_ for more details. (:pull:`6747`) +#. `@stephenworsley`_ made NetCDF loading more efficient by filtering variables + before they become instantiated as cubes in the case where multiple name + constraints are given. This was previously only implemented where one such + constraint was given. (:issue:`6228`, :pull:`6754`) + 🔥 Deprecations ===============