diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 9b9dcff946..35359c4482 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -91,6 +91,11 @@ This document explains the changes made to Iris for this release is hoped that a future ``libnetcdf`` release will recover the original performance. See `netcdf-c#3183`_ for more details. (:pull:`6747`) +#. `@stephenworsley`_ made NetCDF loading more efficient by filtering variables + before they become instantiated as cubes in the case where multiple name + constraints are given. This was previously only implemented where one such + constraint was given. (:issue:`6228`, :pull:`6754`) + 🔥 Deprecations =============== diff --git a/lib/iris/fileformats/netcdf/loader.py b/lib/iris/fileformats/netcdf/loader.py index e8d283beb8..219f681e67 100644 --- a/lib/iris/fileformats/netcdf/loader.py +++ b/lib/iris/fileformats/netcdf/loader.py @@ -601,24 +601,26 @@ def _translate_constraints_to_var_callback(constraints): Notes ----- - For now, ONLY handles a single NameConstraint with no 'STASH' component. + For now, ONLY handles NameConstraints with no 'STASH' component. """ import iris._constraints constraints = iris._constraints.list_of_constraints(constraints) - result = None - if len(constraints) == 1: - (constraint,) = constraints - if ( - isinstance(constraint, iris._constraints.NameConstraint) - and constraint.STASH == "none" - ): - # As long as it doesn't use a STASH match, then we can treat it as - # a testing against name properties of cf_var. - # That's just like testing against name properties of a cube, except that they may not all exist. - def inner(cf_datavar): - match = True + if len(constraints) == 0 or not all( + isinstance(constraint, iris._constraints.NameConstraint) + and constraint.STASH == "none" + for constraint in constraints + ): + # We can define a var-filtering function to speedup the load, *ONLY* when we + # have some constraints, and all are simple NameConstraints with no STASH. + result = None + else: + + def inner(cf_datavar): + match_any_constraint = False + for constraint in constraints: + match_this_constraint = True for name in constraint._names: expected = getattr(constraint, name) if name != "STASH" and expected != "none": @@ -629,11 +631,14 @@ def inner(cf_datavar): continue actual = getattr(cf_datavar, attr_name, "") if actual != expected: - match = False + match_this_constraint = False break - return match + if match_this_constraint: + match_any_constraint = True + break + return match_any_constraint - result = inner + result = inner return result diff --git a/lib/iris/tests/integration/netcdf/test_general.py b/lib/iris/tests/integration/netcdf/test_general.py index 40334463e5..21afb6bc8b 100644 --- a/lib/iris/tests/integration/netcdf/test_general.py +++ b/lib/iris/tests/integration/netcdf/test_general.py @@ -314,6 +314,13 @@ def test_netcdf_with_NameConstraint(self): self.assertEqual(len(cubes), 1) self.assertEqual(cubes[0].var_name, "cdf_temp_dmax_tmean_abs") + def test_netcdf_with_2_NameConstraints(self): + var_names = ["cdf_temp_dmax_tmean_abs", "temp_dmax_tmean_abs"] + constrs = [iris.NameConstraint(var_name=var_name) for var_name in var_names] + cubes = iris.load(self.filename, constrs) + self.assertEqual(len(cubes), 2) + self.assertEqual(sorted([cube.var_name for cube in cubes]), var_names) + def test_netcdf_with_no_constraint(self): cubes = iris.load(self.filename) self.assertEqual(len(cubes), 3) diff --git a/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py b/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py index 64c2c82007..b95bbd0552 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py +++ b/lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py @@ -33,11 +33,35 @@ class Test(tests.IrisTest): def test_multiple_constraints(self): constrs = [ iris.NameConstraint(standard_name="x_wind"), - iris.NameConstraint(var_name="var1"), + iris.NameConstraint(var_name="var2"), + ] + callback = _translate_constraints_to_var_callback(constrs) + result = [callback(var) for var in self.data_variables] + self.assertArrayEqual(result, [True, True, False, True, False]) + + def test_multiple_constraints_invalid(self): + constrs = [ + iris.NameConstraint(standard_name="x_wind"), + iris.NameConstraint(var_name="var1", STASH="m01s00i024"), ] result = _translate_constraints_to_var_callback(constrs) self.assertIsNone(result) + def test_multiple_constraints__multiname(self): + # Modify the first constraint to require BOTH var-name and std-name match + constrs = [ + iris.NameConstraint(standard_name="x_wind", var_name="var1"), + iris.NameConstraint(var_name="var2"), + ] + callback = _translate_constraints_to_var_callback(constrs) + # Add 2 extra vars: one passes both name checks, and the other does not + vars = self.data_variables + [ + CFDataVariable("var1", MagicMock(standard_name="x_wind")), + CFDataVariable("var1", MagicMock(standard_name="air_pressure")), + ] + result = [callback(var) for var in vars] + self.assertArrayEqual(result, [True, True, False, True, False, True, False]) + def test_non_NameConstraint(self): constr = iris.AttributeConstraint(STASH="m01s00i002") result = _translate_constraints_to_var_callback(constr) @@ -91,6 +115,11 @@ def test_NameConstraint_with_STASH(self): result = _translate_constraints_to_var_callback(constr) self.assertIsNone(result) + def test_no_constraints(self): + constrs = [] + result = _translate_constraints_to_var_callback(constrs) + self.assertIsNone(result) + if __name__ == "__main__": tests.main()