From a4a4b5ba0a4ee4a78481ac1ec334483a838c5ffd Mon Sep 17 00:00:00 2001 From: Richard Hattersley Date: Mon, 17 Sep 2012 16:46:16 +0100 Subject: [PATCH 1/2] First cut of hybrid-pressure for GRIB. NB. Now allows transformation of reference cubes. --- lib/iris/etc/grib_cross_reference_rules.txt | 11 +++- lib/iris/etc/grib_rules.txt | 22 ++++--- lib/iris/etc/pp_cross_reference_rules.txt | 2 +- lib/iris/fileformats/rules.py | 71 ++++++++++++++++----- 4 files changed, 78 insertions(+), 28 deletions(-) diff --git a/lib/iris/etc/grib_cross_reference_rules.txt b/lib/iris/etc/grib_cross_reference_rules.txt index 190c4f57b5..ba4ff72792 100644 --- a/lib/iris/etc/grib_cross_reference_rules.txt +++ b/lib/iris/etc/grib_cross_reference_rules.txt @@ -15,4 +15,13 @@ # You should have received a copy of the GNU Lesser General Public License # along with Iris. If not, see . -# PLACEHOLDER +# Equivalent to grib.paramId == 152 +IF +grib.edition == 2 +grib.centre == 'ecmf' +grib.discipline == 0 +grib.parameterCategory == 3 +grib.parameterNumber == 25 +grib.typeOfFirstFixedSurface == 105 +THEN +ReferenceTarget('surface_pressure', lambda cube: {'standard_name': 'surface_air_pressure', 'units': 'Pa', 'data': numpy.exp(cube.data)}) diff --git a/lib/iris/etc/grib_rules.txt b/lib/iris/etc/grib_rules.txt index 72177fa1bb..c5946bc488 100644 --- a/lib/iris/etc/grib_rules.txt +++ b/lib/iris/etc/grib_rules.txt @@ -582,16 +582,18 @@ CoordAndDims(DimCoord(points=0.5*(grib.scaledValueOfFirstFixedSurface/(10.0**gri #ExplicitCoord("model_level", "1", 'z', points=grib.scaledValueOfFirstFixedSurface) #ExplicitCoord('level_height', 'm', 'z', points=grib.pv[grib.scaledValueOfFirstFixedSurface], definitive=True, coord_system=HybridHeightCS(Reference('orography'))) #ExplicitCoord('sigma', '1', 'z', points=grib.pv[grib.numberOfVerticalCoordinateValues/2 + grib.scaledValueOfFirstFixedSurface], coord_system=HybridHeightCS(Reference('orography'))) -# -# hybrid pressure.105 deprecated for 119. -#IF -#grib.edition == 2 -#grib.typeOfFirstFixedSurface in [105, 119] -#grib.numberOfCoordinatesValues > 0 -#THEN -#ExplicitCoord("model_level", "1", 'z', points=grib.scaledValueOfFirstFixedSurface) -#ExplicitCoord('level_height', 'm', 'z', points=grib.pv[grib.scaledValueOfFirstFixedSurface], definitive=True, coord_system=HybridPressureCS(Reference('surface_pressure'))) -#ExplicitCoord('sigma', '1', 'z', points=grib.pv[grib.numberOfPoints/2 + grib.scaledValueOfFirstFixedSurface], coord_system=HybridPressureCS(Reference('surface_pressure'))) + + +# hybrid pressure. 105 deprecated for 119. +IF +grib.edition == 2 +grib.typeOfFirstFixedSurface in [105, 119] +grib.numberOfCoordinatesValues > 0 +THEN +CoordAndDims(AuxCoord(grib.scaledValueOfFirstFixedSurface, standard_name='model_level_number', attributes={'positive': 'up'})) +CoordAndDims(DimCoord(grib.pv[grib.scaledValueOfFirstFixedSurface], long_name='level_pressure', units='Pa')) +CoordAndDims(AuxCoord(grib.pv[grib.numberOfCoordinatesValues/2 + grib.scaledValueOfFirstFixedSurface], long_name='sigma')) +Factory(HybridPressureFactory, [{'long_name': 'level_pressure'}, {'long_name': 'sigma'}, Reference('surface_pressure')]) diff --git a/lib/iris/etc/pp_cross_reference_rules.txt b/lib/iris/etc/pp_cross_reference_rules.txt index 88970d1f7b..1814b4b0ec 100644 --- a/lib/iris/etc/pp_cross_reference_rules.txt +++ b/lib/iris/etc/pp_cross_reference_rules.txt @@ -19,4 +19,4 @@ IF f.lbuser[3] == 33 THEN -Reference('orography',) +ReferenceTarget('orography', None) diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index f437cde8bb..50a89058d7 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -42,6 +42,46 @@ RuleResult = collections.namedtuple('RuleResult', ['cube', 'matching_rules', 'factories']) Factory = collections.namedtuple('Factory', ['factory_class', 'args']) +ReferenceTarget = collections.namedtuple('ReferenceTarget', + ('name', 'transform')) + + +class ConcreteReferenceTarget(object): + """Everything you need to make a real Cube for a named reference.""" + + def __init__(self, name, transform=None): + self.name = name + """The name used to connect references with referencees.""" + self.transform = transform + """An optional transformation to apply to the cubes.""" + self._src_cubes = iris.cube.CubeList() + self._final_cube = None + + def add_cube(self, cube): + self._src_cubes.append(cube) + + def as_cube(self): + if self._final_cube is None: + src_cubes = self._src_cubes + if len(src_cubes) > 1: + # Merge the reference cubes to allow for + # time-varying surface pressure in hybrid-presure. + src_cubes = src_cubes.merge() + if len(src_cubes) > 1: + warnings.warn('Multiple reference cubes for {}' + .format(self.name)) + src_cube = src_cubes[-1] + + if self.transform is None: + self._final_cube = src_cube + else: + final_cube = src_cube.copy() + attributes = self.transform(final_cube) + for name, value in attributes.iteritems(): + setattr(final_cube, name, value) + self._final_cube = final_cube + + return self._final_cube # Controls the deferred import of all the symbols from iris.coords. @@ -343,7 +383,7 @@ def run_actions(self, cube, field): except AttributeError, err: print >> sys.stderr, 'Failed to get value (%(error)s) to execute: %(command)s' % {'command':action, 'error': err} except Exception, err: - print >> sys.stderr, 'Failed (msg:%(error)s) to run: %(command)s\nFrom the rule:%(me)r' % {'me':self, 'command':action, 'error': err} + print >> sys.stderr, 'Failed (msg:%(error)s) to run:\n %(command)s\nFrom the rule:\n%(me)r' % {'me':self, 'command':action, 'error': err} raise err return factories @@ -591,22 +631,13 @@ class _ReferenceError(Exception): pass -def _dereference_args(factory, reference_cubes, regrid_cache, cube): +def _dereference_args(factory, reference_targets, regrid_cache, cube): """Converts all the arguments for a factory into concrete coordinates.""" args = [] for arg in factory.args: if isinstance(arg, iris.fileformats.rules.Reference): - if arg.name in reference_cubes: - # Merge the reference cubes to allow for - # time-varying surface pressure in hybrid-presure. - if len(reference_cubes[arg.name]) > 1: - ref_cubes = iris.cube.CubeList(reference_cubes[arg.name]) - merged = ref_cubes.merge() - if len(merged) > 1: - warnings.warn('Multiple reference cubes for {}' - .format(arg.name)) - reference_cubes[arg.name] = merged[-1:] - src = reference_cubes[arg.name][0] + if arg.name in reference_targets: + src = reference_targets[arg.name].as_cube() # If necessary, regrid the reference cube to # match the grid of this cube. src = _ensure_aligned(regrid_cache, src, cube) @@ -708,7 +739,7 @@ def _ensure_aligned(regrid_cache, src_cube, target_cube): def load_cubes(filenames, user_callback, loader): - reference_cubes = {} + concrete_reference_targets = {} results_needing_reference = [] if isinstance(filenames, basestring): @@ -730,7 +761,15 @@ def load_cubes(filenames, user_callback, loader): rules = loader.cross_ref_rules.matching_rules(field) for rule in rules: reference, = rule.run_actions(cube, field) - reference_cubes.setdefault(reference.name, []).append(cube) + name = reference.name + # Register this cube as a source cube for the named + # reference. + concrete_reference_target = concrete_reference_targets.get(name) + if concrete_reference_target is None: + concrete_reference_target = ConcreteReferenceTarget( + name, reference.transform) + concrete_reference_targets[name] = concrete_reference_target + concrete_reference_target.add_cube(cube) if rules_result.factories: results_needing_reference.append(rules_result) @@ -742,7 +781,7 @@ def load_cubes(filenames, user_callback, loader): cube = result.cube for factory in result.factories: try: - args = _dereference_args(factory, reference_cubes, + args = _dereference_args(factory, concrete_reference_targets, regrid_cache, cube) except _ReferenceError as e: msg = 'Unable to create instance of {factory}. ' + e.message From e4c38e01bedfee29498ae4813e9da3d9728f4dfe Mon Sep 17 00:00:00 2001 From: Richard Hattersley Date: Wed, 19 Sep 2012 20:12:24 +0100 Subject: [PATCH 2/2] New tests, and review tweaks. --- lib/iris/fileformats/rules.py | 13 ++- lib/iris/tests/test_rules.py | 187 ++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 lib/iris/tests/test_rules.py diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index 50a89058d7..4b3d0ddbd1 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -635,7 +635,7 @@ def _dereference_args(factory, reference_targets, regrid_cache, cube): """Converts all the arguments for a factory into concrete coordinates.""" args = [] for arg in factory.args: - if isinstance(arg, iris.fileformats.rules.Reference): + if isinstance(arg, Reference): if arg.name in reference_targets: src = reference_targets[arg.name].as_cube() # If necessary, regrid the reference cube to @@ -764,12 +764,11 @@ def load_cubes(filenames, user_callback, loader): name = reference.name # Register this cube as a source cube for the named # reference. - concrete_reference_target = concrete_reference_targets.get(name) - if concrete_reference_target is None: - concrete_reference_target = ConcreteReferenceTarget( - name, reference.transform) - concrete_reference_targets[name] = concrete_reference_target - concrete_reference_target.add_cube(cube) + target = concrete_reference_targets.get(name) + if target is None: + target = ConcreteReferenceTarget(name, reference.transform) + concrete_reference_targets[name] = target + target.add_cube(cube) if rules_result.factories: results_needing_reference.append(rules_result) diff --git a/lib/iris/tests/test_rules.py b/lib/iris/tests/test_rules.py new file mode 100644 index 0000000000..0fead2a2b3 --- /dev/null +++ b/lib/iris/tests/test_rules.py @@ -0,0 +1,187 @@ +# (C) British Crown Copyright 2010 - 2012, Met Office +# +# This file is part of Iris. +# +# Iris is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Iris is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Iris. If not, see . +""" +Test metadata translation rules. + +""" +# import iris tests first so that some things can be initialised before importing anything else +import iris.tests as tests + +import os +import types + +from iris.aux_factory import HybridHeightFactory +from iris.fileformats.rules import ConcreteReferenceTarget, Factory, Loader, \ + Reference, ReferenceTarget, RuleResult, \ + load_cubes +import iris.tests.stock as stock + + +class Mock(object): + def __repr__(self): + return ''.format(self.__dict__) + + +class TestConcreteReferenceTarget(tests.IrisTest): + def test_attributes(self): + with self.assertRaises(TypeError): + target = ConcreteReferenceTarget() + + target = ConcreteReferenceTarget('foo') + self.assertEqual(target.name, 'foo') + self.assertIsNone(target.transform) + + transform = lambda _: _ + target = ConcreteReferenceTarget('foo', transform) + self.assertEqual(target.name, 'foo') + self.assertIs(target.transform, transform) + + def test_single_cube_no_transform(self): + target = ConcreteReferenceTarget('foo') + src = stock.simple_2d() + target.add_cube(src) + self.assertIs(target.as_cube(), src) + + def test_single_cube_with_transform(self): + transform = lambda cube: {'long_name': 'wibble'} + target = ConcreteReferenceTarget('foo', transform) + src = stock.simple_2d() + target.add_cube(src) + dest = target.as_cube() + self.assertEqual(dest.long_name, 'wibble') + self.assertNotEqual(dest, src) + dest.long_name = src.long_name + self.assertEqual(dest, src) + + def test_multiple_cubes_no_transform(self): + target = ConcreteReferenceTarget('foo') + src = stock.realistic_4d() + for i in range(src.shape[0]): + target.add_cube(src[i]) + dest = target.as_cube() + self.assertIsNot(dest, src) + self.assertEqual(dest, src) + + def test_multiple_cubes_with_transform(self): + transform = lambda cube: {'long_name': 'wibble'} + target = ConcreteReferenceTarget('foo', transform) + src = stock.realistic_4d() + for i in range(src.shape[0]): + target.add_cube(src[i]) + dest = target.as_cube() + self.assertEqual(dest.long_name, 'wibble') + self.assertNotEqual(dest, src) + dest.long_name = src.long_name + self.assertEqual(dest, src) + + +class TestLoadCubes(tests.IrisTest): + def test_simple_factory(self): + # Test the creation process for a factory definition which only + # uses simple dict arguments. + field = Mock() + field_generator = lambda filename: [field] + # A fake rule set returning: + # 1) A parameter cube needing a simple factory construction. + src_cube = Mock() + src_cube.coord = lambda **args: args + src_cube.add_aux_factory = lambda aux_factory: \ + setattr(src_cube, 'fake_aux_factory', aux_factory) + aux_factory = Mock() + factory = Mock() + factory.args = [{'name': 'foo'}] + factory.factory_class = lambda *args: \ + setattr(aux_factory, 'fake_args', args) or aux_factory + rule_result = RuleResult(src_cube, Mock(), [factory]) + rules = Mock() + rules.result = lambda field: rule_result + # A fake cross-reference rule set + xref_rules = Mock() + xref_rules.matching_rules = lambda field: [] + # Finish by making a fake Loader + name = 'FAKE_PP' + fake_loader = Loader(field_generator, rules, xref_rules, name) + cubes = load_cubes(['fake_filename'], None, fake_loader) + # Check the result is a generator with our "cube" as the only + # entry. + self.assertIsInstance(cubes, types.GeneratorType) + cubes = list(cubes) + self.assertEqual(len(cubes), 1) + self.assertIs(cubes[0], src_cube) + # Check the "cube" has an "aux_factory" added, which itself + # must have been created with the correct arguments. + self.assertTrue(hasattr(src_cube, 'fake_aux_factory')) + self.assertIs(src_cube.fake_aux_factory, aux_factory) + self.assertTrue(hasattr(aux_factory, 'fake_args')) + self.assertEqual(aux_factory.fake_args, ({'name': 'foo'},)) + + def test_cross_reference(self): + # Test the creation process for a factory definition which uses + # a cross-reference. + + param_cube = stock.realistic_4d_no_derived() + orog_coord = param_cube.coord('surface_altitude') + param_cube.remove_coord(orog_coord) + + orog_cube = param_cube[0, 0, :, :] + orog_cube.data = orog_coord.points + orog_cube.rename('surface_altitude') + orog_cube.units = orog_coord.units + orog_cube.attributes = orog_coord.attributes + + # We're going to test for the presence of the hybrid height + # stuff later, so let's make sure it's not already there! + assert len(param_cube.aux_factories) == 0 + assert not param_cube.coords('surface_altitude') + + press_field = Mock() + orog_field = Mock() + field_generator = lambda filename: [press_field, orog_field] + # A fake rule set returning: + # 1) A parameter cube needing an "orography" reference + # 2) An "orography" cube + factory = Factory(HybridHeightFactory, [Reference('orography')]) + press_rule_result = RuleResult(param_cube, Mock(), [factory]) + orog_rule_result= RuleResult(orog_cube, Mock(), []) + rules = Mock() + rules.result = lambda field: \ + press_rule_result if field is press_field else orog_rule_result + # A fake cross-reference rule set + ref = ReferenceTarget('orography', None) + orog_xref_rule = Mock() + orog_xref_rule.run_actions = lambda cube, field: (ref,) + xref_rules = Mock() + xref_rules.matching_rules = lambda field: \ + [orog_xref_rule] if field is orog_field else [] + # Finish by making a fake Loader + name = 'FAKE_PP' + fake_loader = Loader(field_generator, rules, xref_rules, name) + cubes = load_cubes(['fake_filename'], None, fake_loader) + # Check the result is a generator containing both of our cubes. + self.assertIsInstance(cubes, types.GeneratorType) + cubes = list(cubes) + self.assertEqual(len(cubes), 2) + self.assertIs(cubes[0], orog_cube) + self.assertIs(cubes[1], param_cube) + # Check the "cube" has an "aux_factory" added, which itself + # must have been created with the correct arguments. + self.assertEqual(len(param_cube.aux_factories), 1) + self.assertEqual(len(param_cube.coords('surface_altitude')), 1) + + +if __name__ == "__main__": + tests.main()