From 325e97a2b7e36ecdad4d21a4bf6e9138cb69f847 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Wed, 28 Jul 2021 15:49:28 +0100 Subject: [PATCH 01/25] Functionality to load meshes independent of cubes. --- lib/iris/experimental/ugrid/__init__.py | 111 +++++++++++++++++++++++- lib/iris/fileformats/cf.py | 5 ++ lib/iris/fileformats/netcdf.py | 12 +-- 3 files changed, 117 insertions(+), 11 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 144c3e4da6..0325bb6eb2 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -15,7 +15,9 @@ from collections.abc import Iterable from contextlib import contextmanager from functools import wraps +from itertools import groupby import logging +from pathlib import Path import re import threading @@ -36,9 +38,14 @@ from ...common.mixin import CFVariableMixin from ...config import get_logger from ...coords import AuxCoord, _DimensionalMetadata -from ...exceptions import ConnectivityNotFoundError, CoordinateNotFoundError +from ...exceptions import ( + ConnectivityNotFoundError, + ConstraintMismatchError, + CoordinateNotFoundError, +) from ...fileformats import cf, netcdf from ...fileformats._nc_load_rules.helpers import get_attr_units, get_names +from ...io import decode_uri, expand_filespecs from ...util import guess_coord_axis __all__ = [ @@ -3274,6 +3281,98 @@ def context(self): PARSE_UGRID_ON_LOAD = ParseUGridOnLoad() +def meshes_from_cf(cf_reader): + # TODO: docstring + + # Mesh instances are shared between file phenomena. + # TODO: more sophisticated Mesh sharing between files. + # TODO: access external Mesh cache? + mesh_vars = cf_reader.cf_group.meshes + meshes = { + name: _build_mesh(cf_reader, var, cf_reader.filename) + for name, var in mesh_vars.items() + } + return meshes + + +def load_mesh(uris, var_name=None): + # TODO: docstring + meshes_result = load_meshes(uris, var_name) + result = [mesh for file in meshes_result.values() for mesh in file] + mesh_count = len(result) + if mesh_count != 1: + message = ( + f"Expecting 1 mesh, but input file(s) produced: {mesh_count} ." + ) + raise ConstraintMismatchError(message) + return result[0] + + +def load_meshes(uris, var_name=None): + # TODO: docstring + # No constraints or callbacks supported - these assume they are operating + # on a Cube. + + from iris.fileformats import FORMAT_AGENT + + # TODO: rationalise UGRID/mesh handling once experimental.ugrid is folded + # into standard behaviour. + + if not PARSE_UGRID_ON_LOAD: + # Explicit behaviour, consistent with netcdf.load_cubes(), rather than + # an invisible assumption. + message = ( + f"PARSE_UGRID_ON_LOAD is {bool(PARSE_UGRID_ON_LOAD)}. Must be " + f"True to enable mesh loading." + ) + raise ValueError(message) + + if isinstance(uris, str): + uris = [uris] + + # Group collections of uris by their iris handler + # Create list of tuples relating schemes to part names. + uri_tuples = sorted(decode_uri(uri) for uri in uris) + + valid_sources = [] + for scheme, groups in groupby(uri_tuples, key=lambda x: x[0]): + # Call each scheme handler with the appropriate URIs + if scheme == "file": + filenames = [x[1] for x in groups] + sources = expand_filespecs(filenames) + elif scheme in ["http", "https"]: + sources = [":".join(x) for x in groups] + else: + message = f"Iris cannot handle the URI scheme: {scheme}" + raise ValueError(message) + + for source in sources: + if scheme == "file": + with open(source, "rb") as fh: + handling_format_spec = FORMAT_AGENT.get_spec( + Path(source).name, fh + ) + else: + handling_format_spec = FORMAT_AGENT.get_spec(source, None) + + if handling_format_spec.handler == netcdf.load_cubes: + valid_sources.append(source) + else: + message = f"Ignoring non-NetCDF file: {source}" + logger.info(msg=message, extra=dict(cls=None)) + + result = {} + for source in valid_sources: + meshes_dict = meshes_from_cf(CFUGridReader(source)) + meshes = meshes_dict.values() + if var_name is not None: + meshes = filter(lambda m: m.var_name == var_name, meshes) + if meshes: + result[source] = list(meshes) + + return result + + ############ # CF Overrides. # These are not included in __all__ since they are not [currently] needed @@ -3469,7 +3568,17 @@ def identify(cls, variables, ignore=None, target=None, warn=True): log_level = logging.WARNING if warn else logging.DEBUG # Identify all CF-UGRID mesh variables. + all_vars = target == variables for nc_var_name, nc_var in target.items(): + if all_vars: + # SPECIAL BEHAVIOUR FOR MESH VARIABLES. + # We are looking for all mesh variables. Check if THIS variable + # is a mesh using its own attributes. + if getattr(nc_var, "cf_role", "") == "mesh_topology": + result[nc_var_name] = CFUGridMeshVariable( + nc_var_name, nc_var + ) + # Check for mesh variable references. nc_var_att = getattr(nc_var, cls.cf_identity, None) diff --git a/lib/iris/fileformats/cf.py b/lib/iris/fileformats/cf.py index 1429e4f65e..c1b23ccb29 100644 --- a/lib/iris/fileformats/cf.py +++ b/lib/iris/fileformats/cf.py @@ -1067,6 +1067,11 @@ def __init__(self, filename, warn=False, monotonic=False): self._build_cf_groups() self._reset() + @property + def filename(self): + # TODO: docstring + return self._filename + def __repr__(self): return "%s(%r)" % (self.__class__.__name__, self._filename) diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index a4727ea624..130794e110 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -792,8 +792,8 @@ def load_cubes(filenames, callback=None): from iris.experimental.ugrid import ( PARSE_UGRID_ON_LOAD, CFUGridReader, - _build_mesh, _build_mesh_coords, + meshes_from_cf, ) from iris.io import run_callback @@ -808,15 +808,7 @@ def load_cubes(filenames, callback=None): meshes = {} if PARSE_UGRID_ON_LOAD: cf = CFUGridReader(filename) - - # Mesh instances are shared between file phenomena. - # TODO: more sophisticated Mesh sharing between files. - # TODO: access external Mesh cache? - mesh_vars = cf.cf_group.meshes - meshes = { - name: _build_mesh(cf, var, filename) - for name, var in mesh_vars.items() - } + meshes = meshes_from_cf(cf) else: cf = iris.fileformats.cf.CFReader(filename) From f9ece8ade9bcdcafefdc60d61f05ecb3d5612a2b Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Wed, 28 Jul 2021 17:10:58 +0100 Subject: [PATCH 02/25] Make meshes_from_cf private. --- lib/iris/experimental/ugrid/__init__.py | 4 ++-- lib/iris/fileformats/netcdf.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 0325bb6eb2..b258788589 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3281,7 +3281,7 @@ def context(self): PARSE_UGRID_ON_LOAD = ParseUGridOnLoad() -def meshes_from_cf(cf_reader): +def _meshes_from_cf(cf_reader): # TODO: docstring # Mesh instances are shared between file phenomena. @@ -3363,7 +3363,7 @@ def load_meshes(uris, var_name=None): result = {} for source in valid_sources: - meshes_dict = meshes_from_cf(CFUGridReader(source)) + meshes_dict = _meshes_from_cf(CFUGridReader(source)) meshes = meshes_dict.values() if var_name is not None: meshes = filter(lambda m: m.var_name == var_name, meshes) diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 130794e110..7bb90665b6 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -793,7 +793,7 @@ def load_cubes(filenames, callback=None): PARSE_UGRID_ON_LOAD, CFUGridReader, _build_mesh_coords, - meshes_from_cf, + _meshes_from_cf, ) from iris.io import run_callback @@ -808,7 +808,7 @@ def load_cubes(filenames, callback=None): meshes = {} if PARSE_UGRID_ON_LOAD: cf = CFUGridReader(filename) - meshes = meshes_from_cf(cf) + meshes = _meshes_from_cf(cf) else: cf = iris.fileformats.cf.CFReader(filename) From 4f4101f9f1bbaa62696c9ef8982543dab24bd7de Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Wed, 28 Jul 2021 22:09:51 +0100 Subject: [PATCH 03/25] Added tests. --- lib/iris/experimental/ugrid/__init__.py | 8 +- .../ugrid/test_CFUGridMeshVariable.py | 41 ++++ .../unit/experimental/ugrid/test_load_mesh.py | 57 +++++ .../experimental/ugrid/test_load_meshes.py | 213 ++++++++++++++++++ 4 files changed, 316 insertions(+), 3 deletions(-) create mode 100644 lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py create mode 100644 lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index b258788589..b53ba1845b 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -52,6 +52,8 @@ "CFUGridReader", "Connectivity", "ConnectivityMetadata", + "load_mesh", + "load_meshes", "Mesh", "Mesh1DConnectivities", "Mesh1DCoords", @@ -3364,11 +3366,11 @@ def load_meshes(uris, var_name=None): result = {} for source in valid_sources: meshes_dict = _meshes_from_cf(CFUGridReader(source)) - meshes = meshes_dict.values() + meshes = list(meshes_dict.values()) if var_name is not None: - meshes = filter(lambda m: m.var_name == var_name, meshes) + meshes = list(filter(lambda m: m.var_name == var_name, meshes)) if meshes: - result[source] = list(meshes) + result[source] = meshes return result diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py index e08dbc769e..d2f806fbbb 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py @@ -31,6 +31,22 @@ class TestIdentify(tests.IrisTest): def setUp(self): self.cf_identity = "mesh" + def test_cf_role(self): + match_name = "match" + match = named_variable(match_name) + setattr(match, "cf_role", "mesh_topology") + + not_match_name = f"not_{match_name}" + not_match = named_variable(not_match_name) + setattr(not_match, "cf_role", "foo") + + vars_all = {match_name: match, not_match_name: not_match} + + # ONLY expecting match, excluding not_match. + expected = {match_name: CFUGridMeshVariable(match_name, match)} + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual(expected, result) + def test_cf_identity(self): subject_name = "ref_subject" ref_subject = named_variable(subject_name) @@ -49,6 +65,31 @@ def test_cf_identity(self): result = CFUGridMeshVariable.identify(vars_all) self.assertDictEqual(expected, result) + def test_cf_role_and_identity(self): + role_match_name = "match" + role_match = named_variable(role_match_name) + setattr(role_match, "cf_role", "mesh_topology") + + subject_name = "ref_subject" + ref_subject = named_variable(subject_name) + ref_source = named_variable("ref_source") + setattr(ref_source, self.cf_identity, subject_name) + + vars_all = { + role_match_name: role_match, + subject_name: ref_subject, + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + # Expecting role_match and ref_subject but excluding other variables. + expected = { + role_match_name: CFUGridMeshVariable(role_match_name, role_match), + subject_name: CFUGridMeshVariable(subject_name, ref_subject), + } + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual(expected, result) + def test_duplicate_refs(self): subject_name = "ref_subject" ref_subject = named_variable(subject_name) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py b/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py new file mode 100644 index 0000000000..151ccc4922 --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py @@ -0,0 +1,57 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :func:`iris.experimental.ugrid.load_mesh` function. + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + +from unittest import mock + +from iris.exceptions import ConstraintMismatchError +from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_mesh + + +class Tests(tests.IrisTest): + # All 'real' tests have been done for load_meshes(). Here we just check + # that load_mesh() works with load_meshes() correctly, using mocking. + def setUp(self): + patcher = mock.patch("iris.experimental.ugrid.load_meshes") + self.addCleanup(patcher.stop) + + self.load_meshes_mock = patcher.start() + # The expected return from load_meshes - a dict of files, each with + # a list of meshes. + self.load_meshes_mock.return_value = {"file": ["mesh"]} + + def test_calls_load_meshes(self): + args = [("file_1", "file_2"), "my_var_name"] + with PARSE_UGRID_ON_LOAD.context(): + _ = load_mesh(args) + self.assertTrue(self.load_meshes_mock.called_with(args)) + + def test_returns_mesh(self): + with PARSE_UGRID_ON_LOAD.context(): + mesh = load_mesh([]) + self.assertEqual(mesh, "mesh") + + def test_single_mesh(self): + # Override the load_meshes_mock return values to provoke errors. + def common(ret_val): + self.load_meshes_mock.return_value = ret_val + with self.assertRaisesRegex( + ConstraintMismatchError, "Expecting 1 mesh.*" + ): + with PARSE_UGRID_ON_LOAD.context(): + _ = load_mesh([]) + + # Too many. + common({"file": ["mesh1", "mesh2"]}) + # Too few. + common({"file": []}) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py new file mode 100644 index 0000000000..09ebd5831d --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py @@ -0,0 +1,213 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :func:`iris.experimental.ugrid.load_meshes` function. + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + +from pathlib import Path +from shutil import rmtree +from subprocess import check_call +import tempfile +from uuid import uuid4 + +from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_meshes, logger + + +def setUpModule(): + global TMP_DIR + TMP_DIR = Path(tempfile.mkdtemp()) + + +def tearDownModule(): + if TMP_DIR is not None: + rmtree(TMP_DIR) + + +def cdl_to_nc(cdl): + cdl_path = TMP_DIR / "tst.cdl" + nc_path = TMP_DIR / f"{uuid4()}.nc" + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + return str(nc_path) + + +class Tests(tests.IrisTest): + def setUp(self): + self.ref_cdl = """ + netcdf mesh_test { + dimensions: + node = 3 ; + face = 1 ; + vertex = 3 ; + levels = 2 ; + variables: + int mesh ; + mesh:cf_role = "mesh_topology" ; + mesh:topology_dimension = 2 ; + mesh:node_coordinates = "node_x node_y" ; + mesh:face_coordinates = "face_x face_y" ; + mesh:face_node_connectivity = "face_nodes" ; + float node_x(node) ; + node_x:standard_name = "longitude" ; + float node_y(node) ; + node_y:standard_name = "latitude" ; + float face_x(face) ; + face_x:standard_name = "longitude" ; + float face_y(face) ; + face_y:standard_name = "latitude" ; + int face_nodes(face, vertex) ; + face_nodes:cf_role = "face_node_connectivity" ; + face_nodes:start_index = 0 ; + int levels(levels) ; + float node_data(levels, node) ; + node_data:coordinates = "node_x node_y" ; + node_data:location = "node" ; + node_data:mesh = "mesh" ; + float face_data(levels, face) ; + face_data:coordinates = "face_x face_y" ; + face_data:location = "face" ; + face_data:mesh = "mesh" ; + data: + mesh = 0; + node_x = 0., 2., 1.; + node_y = 0., 0., 1.; + face_x = 0.5; + face_y = 0.5; + face_nodes = 0, 1, 2; + levels = 1, 2; + node_data = 0., 0., 0.; + face_data = 0.; + } + """ + self.nc_path = cdl_to_nc(self.ref_cdl) + + def test_with_data(self): + nc_path = cdl_to_nc(self.ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(1, len(file_meshes)) + mesh = file_meshes[0] + self.assertEqual("mesh", mesh.var_name) + + def test_no_data(self): + cdl_lines = self.ref_cdl.split("\n") + cdl_lines = filter( + lambda line: ':mesh = "mesh"' not in line, cdl_lines + ) + ref_cdl = "\n".join(cdl_lines) + + nc_path = cdl_to_nc(ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(1, len(file_meshes)) + mesh = file_meshes[0] + self.assertEqual("mesh", mesh.var_name) + + def test_var_name(self): + nc_path = cdl_to_nc(self.ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path, "some_other_mesh") + self.assertDictEqual({}, meshes) + + def test_multi_files(self): + files_count = 3 + nc_paths = [cdl_to_nc(self.ref_cdl) for _ in range(files_count)] + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_paths) + self.assertEqual(files_count, len(meshes)) + + def test_multi_meshes(self): + cdl_extra = """ + int mesh2 ; + mesh2:cf_role = "mesh_topology" ; + mesh2:topology_dimension = 2 ; + mesh2:node_coordinates = "node_x node_y" ; + mesh2:face_coordinates = "face_x face_y" ; + mesh2:face_node_connectivity = "face_nodes" ; + """ + vars_string = "variables:" + vars_start = self.ref_cdl.index(vars_string) + len(vars_string) + ref_cdl = ( + self.ref_cdl[:vars_start] + cdl_extra + self.ref_cdl[vars_start:] + ) + + nc_path = cdl_to_nc(ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(2, len(file_meshes)) + mesh_names = [mesh.var_name for mesh in file_meshes] + self.assertIn("mesh", mesh_names) + self.assertIn("mesh2", mesh_names) + + def test_no_parsing(self): + nc_path = cdl_to_nc(self.ref_cdl) + with self.assertRaisesRegex( + ValueError, ".*Must be True to enable mesh loading." + ): + _ = load_meshes(nc_path) + + def test_invalid_scheme(self): + with self.assertRaisesRegex( + ValueError, "Iris cannot handle the URI scheme:.*" + ): + with PARSE_UGRID_ON_LOAD.context(): + _ = load_meshes("foo://bar") + + def test_http(self): + # Are we OK to rely on a 3rd party URL? + # Should we instead be hosting a UGRID file over OpenDAP for testing? + # Fairly slow. + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes( + "http://amb6400b.stccmop.org:8080/thredds/dodsC/model_data/forecast" + ) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(1, len(file_meshes)) + mesh = file_meshes[0] + self.assertEqual("Mesh", mesh.var_name) + + def test_mixed_sources(self): + URL = "http://amb6400b.stccmop.org:8080/thredds/dodsC/model_data/forecast" + file = cdl_to_nc(self.ref_cdl) + glob = f"{TMP_DIR}/*.nc" + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes([URL, glob]) + for source in (URL, file): + self.assertIn(source, meshes) + + @tests.skip_data + def test_non_nc(self): + log_regex = r"Ignoring non-NetCDF file:.*" + with self.assertLogs(logger, level="INFO", msg_regex=log_regex): + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes( + tests.get_data_path(["PP", "simple_pp", "global.pp"]) + ) + self.assertDictEqual({}, meshes) From 6f49fd25f786296ffd697a946665f1921e634ae2 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Thu, 29 Jul 2021 10:20:00 +0100 Subject: [PATCH 04/25] Mesh loading docstrings. --- lib/iris/experimental/ugrid/__init__.py | 42 +++++++++++++++++++++++-- lib/iris/fileformats/cf.py | 2 +- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index b53ba1845b..ffbda1b33e 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3284,8 +3284,12 @@ def context(self): def _meshes_from_cf(cf_reader): - # TODO: docstring + """ + Common behaviour for extracting meshes from a CFReader. + + Simple now, but expected to increase in complexity as Mesh sharing develops. + """ # Mesh instances are shared between file phenomena. # TODO: more sophisticated Mesh sharing between files. # TODO: access external Mesh cache? @@ -3298,7 +3302,23 @@ def _meshes_from_cf(cf_reader): def load_mesh(uris, var_name=None): - # TODO: docstring + """ + Create a single :class:`Mesh` object from one or more NetCDF files. + + Raises an error if more/less than one :class:`Mesh` is found. + + Parameters + ---------- + uris : str or iterable of str + One or more filenames/URI's. Any URI's must support OpenDAP. + var_name : str, optional + Only return a :class:`Mesh` if its var_name matches this value. + + Returns + ------- + :class:`Mesh` + + """ meshes_result = load_meshes(uris, var_name) result = [mesh for file in meshes_result.values() for mesh in file] mesh_count = len(result) @@ -3311,7 +3331,23 @@ def load_mesh(uris, var_name=None): def load_meshes(uris, var_name=None): - # TODO: docstring + """ + Create :class:`Mesh` objects from one or more NetCDF files. + + Parameters + ---------- + uris : str or iterable of str + One or more filenames/URI's. Any URI's must support OpenDAP. + var_name : str, optional + Only return a :class:`Mesh` if its var_name matches this value. + + Returns + ------- + dict + A dictionary of file paths/URL's and lists of the :class:`Mesh`es + returned from each. + + """ # No constraints or callbacks supported - these assume they are operating # on a Cube. diff --git a/lib/iris/fileformats/cf.py b/lib/iris/fileformats/cf.py index c1b23ccb29..b22fbd3b51 100644 --- a/lib/iris/fileformats/cf.py +++ b/lib/iris/fileformats/cf.py @@ -1069,7 +1069,7 @@ def __init__(self, filename, warn=False, monotonic=False): @property def filename(self): - # TODO: docstring + """The file that the CFReader is reading.""" return self._filename def __repr__(self): From 6c966ed3128ddf91710adbd95673770c1b23304b Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Thu, 29 Jul 2021 17:57:37 +0100 Subject: [PATCH 05/25] load_mesh integration test. --- .../experimental/test_ugrid_load.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/iris/tests/integration/experimental/test_ugrid_load.py b/lib/iris/tests/integration/experimental/test_ugrid_load.py index 1503225d6f..b852c08ee4 100644 --- a/lib/iris/tests/integration/experimental/test_ugrid_load.py +++ b/lib/iris/tests/integration/experimental/test_ugrid_load.py @@ -14,7 +14,12 @@ from collections.abc import Iterable from iris import Constraint, load -from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, logger +from iris.experimental.ugrid import ( + PARSE_UGRID_ON_LOAD, + Mesh, + load_mesh, + logger, +) # Import iris.tests first so that some things can be initialised before # importing anything else. @@ -179,5 +184,21 @@ def test_mesh_no_topology_dimension(self): self.assertEqual(cube.mesh.topology_dimension, 2) +@tests.skip_data +class Test_load_mesh(tests.IrisTest): + def test_load_mesh(self): + file_path = tests.get_data_path( + [ + "NetCDF", + "unstructured_grid", + "lfric_ngvat_2D_1t_face_half_levels_main_conv_rain.nc", + ] + ) + with PARSE_UGRID_ON_LOAD.context(): + mesh = load_mesh(file_path) + # Can't use a CML test as this isn't supported for non-Cubes. + self.assertIsInstance(mesh, Mesh) + + if __name__ == "__main__": tests.main() From df01b3f9d1a2dee3d159f50dd2167b2706578f5f Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 30 Jul 2021 15:53:22 +0100 Subject: [PATCH 06/25] Testing improvements. --- .cirrus.yml | 4 +- lib/iris/experimental/ugrid/__init__.py | 5 +- .../experimental/test_ugrid_load.py | 33 ++++++----- .../experimental/ugrid/test_load_meshes.py | 57 ++++++++++--------- 4 files changed, 54 insertions(+), 45 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index e55b69dc69..e6bd0fc424 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -38,7 +38,7 @@ env: # Conda packages to be installed. CONDA_CACHE_PACKAGES: "nox pip" # Git commit hash for iris test data. - IRIS_TEST_DATA_VERSION: "2.2" + IRIS_TEST_DATA_VERSION: "2.4" # Base directory for the iris-test-data. IRIS_TEST_DATA_DIR: ${HOME}/iris-test-data @@ -193,4 +193,4 @@ task: - mkdir -p ${MPL_RC_DIR} - echo "backend : agg" > ${MPL_RC_FILE} - echo "image.cmap : viridis" >> ${MPL_RC_FILE} - - nox --session linkcheck -- --verbose \ No newline at end of file + - nox --session linkcheck -- --verbose diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index ffbda1b33e..882890afc1 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3348,14 +3348,13 @@ def load_meshes(uris, var_name=None): returned from each. """ + # TODO: rationalise UGRID/mesh handling once experimental.ugrid is folded + # into standard behaviour. # No constraints or callbacks supported - these assume they are operating # on a Cube. from iris.fileformats import FORMAT_AGENT - # TODO: rationalise UGRID/mesh handling once experimental.ugrid is folded - # into standard behaviour. - if not PARSE_UGRID_ON_LOAD: # Explicit behaviour, consistent with netcdf.load_cubes(), rather than # an invisible assumption. diff --git a/lib/iris/tests/integration/experimental/test_ugrid_load.py b/lib/iris/tests/integration/experimental/test_ugrid_load.py index b852c08ee4..483fd435d4 100644 --- a/lib/iris/tests/integration/experimental/test_ugrid_load.py +++ b/lib/iris/tests/integration/experimental/test_ugrid_load.py @@ -11,6 +11,10 @@ """ +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + from collections.abc import Iterable from iris import Constraint, load @@ -20,10 +24,6 @@ load_mesh, logger, ) - -# Import iris.tests first so that some things can be initialised before -# importing anything else. -import iris.tests as tests from iris.tests.stock.netcdf import ( _file_from_cdl_template as create_file_from_cdl_template, ) @@ -186,18 +186,23 @@ def test_mesh_no_topology_dimension(self): @tests.skip_data class Test_load_mesh(tests.IrisTest): - def test_load_mesh(self): - file_path = tests.get_data_path( - [ - "NetCDF", - "unstructured_grid", - "lfric_ngvat_2D_1t_face_half_levels_main_conv_rain.nc", - ] - ) + def common_test(self, file_name, mesh_var_name): with PARSE_UGRID_ON_LOAD.context(): - mesh = load_mesh(file_path) - # Can't use a CML test as this isn't supported for non-Cubes. + mesh = load_mesh( + tests.get_data_path(["NetCDF", "unstructured_grid", file_name]) + ) + # NOTE: cannot use CML tests as this isn't supported for non-Cubes. self.assertIsInstance(mesh, Mesh) + self.assertEqual(mesh.var_name, mesh_var_name) + + def test_full_file(self): + self.common_test( + "lfric_ngvat_2D_1t_face_half_levels_main_conv_rain.nc", + "Mesh2d_half_levels", + ) + + def test_mesh_file(self): + self.common_test("mesh_C12.nc", "dynamics") if __name__ == "__main__": diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py index 09ebd5831d..f4ae324ecd 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py @@ -16,6 +16,7 @@ from shutil import rmtree from subprocess import check_call import tempfile +from unittest import mock from uuid import uuid4 from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_meshes, logger @@ -43,7 +44,7 @@ def cdl_to_nc(cdl): return str(nc_path) -class Tests(tests.IrisTest): +class TestsBasic(tests.IrisTest): def setUp(self): self.ref_cdl = """ netcdf mesh_test { @@ -177,31 +178,6 @@ def test_invalid_scheme(self): with PARSE_UGRID_ON_LOAD.context(): _ = load_meshes("foo://bar") - def test_http(self): - # Are we OK to rely on a 3rd party URL? - # Should we instead be hosting a UGRID file over OpenDAP for testing? - # Fairly slow. - with PARSE_UGRID_ON_LOAD.context(): - meshes = load_meshes( - "http://amb6400b.stccmop.org:8080/thredds/dodsC/model_data/forecast" - ) - - files = list(meshes.keys()) - self.assertEqual(1, len(files)) - file_meshes = meshes[files[0]] - self.assertEqual(1, len(file_meshes)) - mesh = file_meshes[0] - self.assertEqual("Mesh", mesh.var_name) - - def test_mixed_sources(self): - URL = "http://amb6400b.stccmop.org:8080/thredds/dodsC/model_data/forecast" - file = cdl_to_nc(self.ref_cdl) - glob = f"{TMP_DIR}/*.nc" - with PARSE_UGRID_ON_LOAD.context(): - meshes = load_meshes([URL, glob]) - for source in (URL, file): - self.assertIn(source, meshes) - @tests.skip_data def test_non_nc(self): log_regex = r"Ignoring non-NetCDF file:.*" @@ -211,3 +187,32 @@ def test_non_nc(self): tests.get_data_path(["PP", "simple_pp", "global.pp"]) ) self.assertDictEqual({}, meshes) + + +class TestsHttp(tests.IrisTest): + # Tests of HTTP (OpenDAP) loading need mocking since we can't have tests + # that rely on 3rd party servers. + def setUp(self): + patcher = mock.patch("iris.fileformats.FORMAT_AGENT.get_spec") + self.addCleanup(patcher.stop) + self.format_agent_mock = patcher.start() + + def test_http(self): + url = "http://foo" + with PARSE_UGRID_ON_LOAD.context(): + _ = load_meshes(url) + self.format_agent_mock.assert_called_with(url, None) + + def test_mixed_sources(self): + url = "http://foo" + file = TMP_DIR / f"{uuid4()}.nc" + file.touch() + glob = f"{TMP_DIR}/*.nc" + + with PARSE_UGRID_ON_LOAD.context(): + _ = load_meshes([url, glob]) + file_uris = [ + call.args[0] for call in self.format_agent_mock.call_args_list + ] + for source in (url, Path(file).name): + self.assertIn(source, file_uris) From 48c6d145b9ccfb144c9d67b1765c5e792e942fd6 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 30 Jul 2021 17:20:14 +0100 Subject: [PATCH 07/25] Mesh bad cf_role tolerance. --- lib/iris/experimental/ugrid/__init__.py | 16 ++++++++++++++- .../experimental/test_ugrid_load.py | 20 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 882890afc1..81987130f0 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3825,6 +3825,21 @@ def _build_mesh(cf, mesh_var, file_path): attributes = {} attr_units = get_attr_units(mesh_var, attributes) + cf_role_message = None + if not hasattr(mesh_var, "cf_role"): + cf_role_message = f"{mesh_var.cf_name} has no cf_role attribute." + cf_role = "mesh_topology" + else: + cf_role = getattr(mesh_var, "cf_role") + if cf_role != "mesh_topology": + cf_role_message = ( + f"{mesh_var.cf_name} has an inappropriate cf_role: {cf_role}." + ) + if cf_role_message: + cf_role_message += " Correcting to 'mesh_topology'." + # TODO: reconsider logging level when we have consistent practice. + logger.warning(cf_role_message, extra=dict(cls=None)) + if hasattr(mesh_var, "volume_node_connectivity"): topology_dimension = 3 elif hasattr(mesh_var, "face_node_connectivity"): @@ -3928,7 +3943,6 @@ def _build_mesh(cf, mesh_var, file_path): edge_dimension=edge_dimension, face_dimension=face_dimension, ) - assert mesh.cf_role == mesh_var.cf_role mesh_elements = ( list(mesh.all_coords) + list(mesh.all_connectivities) + [mesh] diff --git a/lib/iris/tests/integration/experimental/test_ugrid_load.py b/lib/iris/tests/integration/experimental/test_ugrid_load.py index 483fd435d4..f796c18f06 100644 --- a/lib/iris/tests/integration/experimental/test_ugrid_load.py +++ b/lib/iris/tests/integration/experimental/test_ugrid_load.py @@ -183,6 +183,26 @@ def test_mesh_no_topology_dimension(self): # Check that the result has the correct topology-dimension value. self.assertEqual(cube.mesh.topology_dimension, 2) + def test_mesh_bad_cf_role(self): + # Check that the load generates a suitable warning. + log_regex = r"inappropriate cf_role" + with self.assertLogs(logger, level="WARNING", msg_regex=log_regex): + template = "minimal_bad_mesh_cf_role" + dim_line = 'mesh_var:cf_role = "foo" ;' + _ = self.create_synthetic_test_cube( + template=template, subs=dict(CF_ROLE_DEFINITION=dim_line) + ) + + def test_mesh_no_cf_role(self): + # Check that the load generates a suitable warning. + log_regex = r"no cf_role attribute" + with self.assertLogs(logger, level="WARNING", msg_regex=log_regex): + template = "minimal_bad_mesh_cf_role" + dim_line = "" + _ = self.create_synthetic_test_cube( + template=template, subs=dict(CF_ROLE_DEFINITION=dim_line) + ) + @tests.skip_data class Test_load_mesh(tests.IrisTest): From 5a29d44a8f40d3090fdf8b65a3e475efa1b14b81 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 30 Jul 2021 17:22:20 +0100 Subject: [PATCH 08/25] load_mesh raise ValueError. --- lib/iris/experimental/ugrid/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 81987130f0..4d0f6bb826 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -38,11 +38,7 @@ from ...common.mixin import CFVariableMixin from ...config import get_logger from ...coords import AuxCoord, _DimensionalMetadata -from ...exceptions import ( - ConnectivityNotFoundError, - ConstraintMismatchError, - CoordinateNotFoundError, -) +from ...exceptions import ConnectivityNotFoundError, CoordinateNotFoundError from ...fileformats import cf, netcdf from ...fileformats._nc_load_rules.helpers import get_attr_units, get_names from ...io import decode_uri, expand_filespecs @@ -3326,7 +3322,7 @@ def load_mesh(uris, var_name=None): message = ( f"Expecting 1 mesh, but input file(s) produced: {mesh_count} ." ) - raise ConstraintMismatchError(message) + raise ValueError(message) return result[0] From 8a981a7997008e3ec815a1f04d8a5b69cd19b995 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 30 Jul 2021 17:24:39 +0100 Subject: [PATCH 09/25] Better var_name docstring for load_meshes. --- lib/iris/experimental/ugrid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 4d0f6bb826..d0a7d4aa8c 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3335,7 +3335,7 @@ def load_meshes(uris, var_name=None): uris : str or iterable of str One or more filenames/URI's. Any URI's must support OpenDAP. var_name : str, optional - Only return a :class:`Mesh` if its var_name matches this value. + Only return :class:`Mesh`es that have var_names matching this value. Returns ------- From fa5937fcc01bf41d86fb58ad1c6409dd598cbafa Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 30 Jul 2021 17:29:54 +0100 Subject: [PATCH 10/25] Mesh load testing tidy-up. --- .../experimental/test_ugrid_load.py | 1 - .../file_headers/minimal_bad_mesh_cf_role.cdl | 38 +++++++++++++++++++ .../ugrid/test_CFUGridMeshVariable.py | 8 ++-- .../unit/experimental/ugrid/test_load_mesh.py | 5 +-- 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 lib/iris/tests/stock/file_headers/minimal_bad_mesh_cf_role.cdl diff --git a/lib/iris/tests/integration/experimental/test_ugrid_load.py b/lib/iris/tests/integration/experimental/test_ugrid_load.py index f796c18f06..7f6b79188f 100644 --- a/lib/iris/tests/integration/experimental/test_ugrid_load.py +++ b/lib/iris/tests/integration/experimental/test_ugrid_load.py @@ -10,7 +10,6 @@ standard behaviour. """ - # Import iris.tests first so that some things can be initialised before # importing anything else. import iris.tests as tests # isort:skip diff --git a/lib/iris/tests/stock/file_headers/minimal_bad_mesh_cf_role.cdl b/lib/iris/tests/stock/file_headers/minimal_bad_mesh_cf_role.cdl new file mode 100644 index 0000000000..6fb90fd8c6 --- /dev/null +++ b/lib/iris/tests/stock/file_headers/minimal_bad_mesh_cf_role.cdl @@ -0,0 +1,38 @@ +// Tolerant loading test example : the mesh has the wrong 'mesh_topology' +// NOTE: *not* truly minimal, as we cannot (yet) handle data with no face coords. +netcdf ${DATASET_NAME} { +dimensions: + NODES = ${NUM_NODES} ; + FACES = ${NUM_FACES} ; + FACE_CORNERS = 4 ; +variables: + int mesh_var ; + ${CF_ROLE_DEFINITION} + mesh_var:topology_dimension = 2 ; + mesh_var:node_coordinates = "mesh_node_x mesh_node_y" ; + mesh_var:face_node_connectivity = "mesh_face_nodes" ; + mesh_var:face_coordinates = "mesh_face_x mesh_face_y" ; + float mesh_node_x(NODES) ; + mesh_node_x:standard_name = "longitude" ; + mesh_node_x:long_name = "Longitude of mesh nodes." ; + mesh_node_x:units = "degrees_east" ; + float mesh_node_y(NODES) ; + mesh_node_y:standard_name = "latitude" ; + mesh_node_y:long_name = "Latitude of mesh nodes." ; + mesh_node_y:units = "degrees_north" ; + float mesh_face_x(FACES) ; + mesh_face_x:standard_name = "longitude" ; + mesh_face_x:long_name = "Longitude of mesh nodes." ; + mesh_face_x:units = "degrees_east" ; + float mesh_face_y(FACES) ; + mesh_face_y:standard_name = "latitude" ; + mesh_face_y:long_name = "Latitude of mesh nodes." ; + mesh_face_y:units = "degrees_north" ; + int mesh_face_nodes(FACES, FACE_CORNERS) ; + mesh_face_nodes:cf_role = "face_node_connectivity" ; + mesh_face_nodes:long_name = "Maps every face to its corner nodes." ; + mesh_face_nodes:start_index = 0 ; + float data_var(FACES) ; + data_var:mesh = "mesh_var" ; + data_var:location = "face" ; +} diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py index d2f806fbbb..f341ac5756 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py @@ -10,13 +10,13 @@ standard behaviour. """ +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + import numpy as np from iris.experimental.ugrid import CFUGridMeshVariable, logger - -# Import iris.tests first so that some things can be initialised before -# importing anything else. -import iris.tests as tests from iris.tests.unit.experimental.ugrid.test_CFUGridReader import ( netcdf_ugrid_variable, ) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py b/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py index 151ccc4922..da70502d78 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py @@ -14,7 +14,6 @@ from unittest import mock -from iris.exceptions import ConstraintMismatchError from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_mesh @@ -45,9 +44,7 @@ def test_single_mesh(self): # Override the load_meshes_mock return values to provoke errors. def common(ret_val): self.load_meshes_mock.return_value = ret_val - with self.assertRaisesRegex( - ConstraintMismatchError, "Expecting 1 mesh.*" - ): + with self.assertRaisesRegex(ValueError, "Expecting 1 mesh.*"): with PARSE_UGRID_ON_LOAD.context(): _ = load_mesh([]) From 6da332d1530c443476039a33c916cbf0a6eb4eb7 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 30 Jul 2021 17:40:19 +0100 Subject: [PATCH 11/25] load_meshes docstring pluralisation fix. --- lib/iris/experimental/ugrid/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index d0a7d4aa8c..87e834d945 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3335,12 +3335,12 @@ def load_meshes(uris, var_name=None): uris : str or iterable of str One or more filenames/URI's. Any URI's must support OpenDAP. var_name : str, optional - Only return :class:`Mesh`es that have var_names matching this value. + Only return :class:`Mesh`'s that have var_names matching this value. Returns ------- dict - A dictionary of file paths/URL's and lists of the :class:`Mesh`es + A dictionary of file paths/URL's and lists of the :class:`Mesh`'s returned from each. """ From e6649c178ba996c68b517187aee35220ba70dee8 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 30 Jul 2021 17:50:12 +0100 Subject: [PATCH 12/25] load_meshes http test py37 compatibility. --- lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py index f4ae324ecd..d82be68b10 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py @@ -212,7 +212,7 @@ def test_mixed_sources(self): with PARSE_UGRID_ON_LOAD.context(): _ = load_meshes([url, glob]) file_uris = [ - call.args[0] for call in self.format_agent_mock.call_args_list + call[0][0] for call in self.format_agent_mock.call_args_list ] for source in (url, Path(file).name): self.assertIn(source, file_uris) From 7316ec62fbf40b07006814c0f1c99f39ef67ad59 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 11:08:59 +0100 Subject: [PATCH 13/25] Correct Sphinx domain pluralisation. --- lib/iris/experimental/ugrid/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 87e834d945..a8a23fac66 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -205,9 +205,9 @@ def __init__( for Fortran and legacy NetCDF files). * src_dim (int): Either ``0`` or ``1``. Default is ``0``. Denotes which dimension - of :attr:`indices` varies over the :attr:`src_location`'s (the + of :attr:`indices` varies over the :attr:`src_location`\\ s (the alternate dimension therefore varying within individual - :attr:`src_location`'s). (This parameter allows support for fastest varying index being + :attr:`src_location`\\ s). (This parameter allows support for fastest varying index being either first or last). E.g. for ``face_node_connectivity``, for 10 faces: ``indices.shape[src_dim] = 10``. @@ -358,7 +358,7 @@ def start_index(self): def src_dim(self): """ The dimension of the connectivity's :attr:`indices` array that varies - over the connectivity's :attr:`src_location`'s. Either ``0`` or ``1``. + over the connectivity's :attr:`src_location`\\ s. Either ``0`` or ``1``. **Read-only** - validity of :attr:`indices` is dependent on :attr:`src_dim`. Use :meth:`transpose` to create a new, transposed :class:`Connectivity` if a different :attr:`src_dim` is needed. @@ -372,7 +372,7 @@ def tgt_dim(self): Derived as the alternate value of :attr:`src_dim` - each must equal either ``0`` or ``1``. The dimension of the connectivity's :attr:`indices` array that varies - within the connectivity's individual :attr:`src_location`'s. + within the connectivity's individual :attr:`src_location`\\ s. """ return self._tgt_dim @@ -496,7 +496,7 @@ def validate_indices(self): """ Perform a thorough validity check of this connectivity's :attr:`indices`. Includes checking the sizes of individual - :attr:`src_location`'s (specified using masks on the + :attr:`src_location`\\ s (specified using masks on the :attr:`indices` array) against the :attr:`cf_role`. Raises a ``ValueError`` if any problems are encountered, otherwise @@ -1933,7 +1933,7 @@ def to_MeshCoord(self, location, axis): def to_MeshCoords(self, location): """ - Generate a tuple of :class:`MeshCoord`'s, each referencing the current + Generate a tuple of :class:`MeshCoord`\\ s, each referencing the current :class:`Mesh`, one for each :attr:`AXES` value, passing through the ``location`` argument. @@ -1947,7 +1947,7 @@ def to_MeshCoords(self, location): The ``location`` argument for :class:`MeshCoord` instantiation. Returns: - tuple of :class:`MeshCoord`'s referencing the current :class:`Mesh`. + tuple of :class:`MeshCoord`\\ s referencing the current :class:`Mesh`. One for each value in :attr:`AXES`, using the value for the ``axis`` argument. @@ -3335,12 +3335,12 @@ def load_meshes(uris, var_name=None): uris : str or iterable of str One or more filenames/URI's. Any URI's must support OpenDAP. var_name : str, optional - Only return :class:`Mesh`'s that have var_names matching this value. + Only return :class:`Mesh`\\ es that have var_names matching this value. Returns ------- dict - A dictionary of file paths/URL's and lists of the :class:`Mesh`'s + A dictionary of file paths/URL's and lists of the :class:`Mesh`\\ es returned from each. """ From 5ffc3d6606f71c0a1efde3d28fa36d25ad6bb4d9 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 11:12:16 +0100 Subject: [PATCH 14/25] Clearer load_meshes Returns docstring. --- lib/iris/experimental/ugrid/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index a8a23fac66..d50f8d0777 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3340,8 +3340,8 @@ def load_meshes(uris, var_name=None): Returns ------- dict - A dictionary of file paths/URL's and lists of the :class:`Mesh`\\ es - returned from each. + A dictionary mapping each file path/URL to a list of the + :class:`Mesh`\\ es returned from each. """ # TODO: rationalise UGRID/mesh handling once experimental.ugrid is folded From a14317b686e6ef32629006c2c9aa6f25309c485a Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 11:38:49 +0100 Subject: [PATCH 15/25] Add no_mesh integration tests. --- .../experimental/test_ugrid_load.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/iris/tests/integration/experimental/test_ugrid_load.py b/lib/iris/tests/integration/experimental/test_ugrid_load.py index 7f6b79188f..5d40b1e69b 100644 --- a/lib/iris/tests/integration/experimental/test_ugrid_load.py +++ b/lib/iris/tests/integration/experimental/test_ugrid_load.py @@ -21,6 +21,7 @@ PARSE_UGRID_ON_LOAD, Mesh, load_mesh, + load_meshes, logger, ) from iris.tests.stock.netcdf import ( @@ -113,6 +114,15 @@ def test_3D_veg_pseudo_levels(self): "3D_veg_pseudo_levels.cml", ) + def test_no_mesh(self): + with PARSE_UGRID_ON_LOAD.context(): + cube_list = load( + tests.get_data_path( + ["NetCDF", "unstructured_grid", "theta_nodal_not_ugrid.nc"] + ) + ) + self.assertTrue(all([cube.mesh is None for cube in cube_list])) + @tests.skip_data class TestMultiplePhenomena(tests.IrisTest): @@ -223,6 +233,15 @@ def test_full_file(self): def test_mesh_file(self): self.common_test("mesh_C12.nc", "dynamics") + def test_no_mesh(self): + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes( + tests.get_data_path( + ["NetCDF", "unstructured_grid", "theta_nodal_not_ugrid.nc"] + ) + ) + self.assertDictEqual({}, meshes) + if __name__ == "__main__": tests.main() From ff73cb0308cba274c1ab50af990cf2dd835c4833 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 11:59:02 +0100 Subject: [PATCH 16/25] Clearer test_CFUGridMeshVariable comments. --- .../ugrid/test_CFUGridMeshVariable.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py index f341ac5756..8736a82fc1 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py @@ -32,6 +32,7 @@ def setUp(self): self.cf_identity = "mesh" def test_cf_role(self): + # Test that mesh variables can be identified by having `cf_role="mesh_topology"`. match_name = "match" match = named_variable(match_name) setattr(match, "cf_role", "mesh_topology") @@ -48,6 +49,8 @@ def test_cf_role(self): self.assertDictEqual(expected, result) def test_cf_identity(self): + # Test that mesh variables can be identified by being another variable's + # `mesh` attribute. subject_name = "ref_subject" ref_subject = named_variable(subject_name) ref_source = named_variable("ref_source") @@ -66,20 +69,26 @@ def test_cf_identity(self): self.assertDictEqual(expected, result) def test_cf_role_and_identity(self): + # Test that identification can successfully handle a combination of + # mesh variables having `cf_role="mesh_topology"` AND being referenced as + # another variable's `mesh` attribute. role_match_name = "match" role_match = named_variable(role_match_name) setattr(role_match, "cf_role", "mesh_topology") + ref_source_1 = named_variable("ref_source_1") + setattr(ref_source_1, self.cf_identity, role_match_name) subject_name = "ref_subject" ref_subject = named_variable(subject_name) - ref_source = named_variable("ref_source") - setattr(ref_source, self.cf_identity, subject_name) + ref_source_2 = named_variable("ref_source_2") + setattr(ref_source_2, self.cf_identity, subject_name) vars_all = { role_match_name: role_match, subject_name: ref_subject, "ref_not_subject": named_variable("ref_not_subject"), - "ref_source": ref_source, + "ref_source_1": ref_source_1, + "ref_source_2": ref_source_2, } # Expecting role_match and ref_subject but excluding other variables. From 436f5f737bc1c1b8fc08631717bcf64fab5f054e Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 12:08:29 +0100 Subject: [PATCH 17/25] Mesh load unit testing use IrisTest.patch(). --- lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py | 9 +++------ .../tests/unit/experimental/ugrid/test_load_meshes.py | 7 +++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py b/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py index da70502d78..8bb78929f8 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py @@ -12,8 +12,6 @@ # importing anything else. import iris.tests as tests # isort:skip -from unittest import mock - from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_mesh @@ -21,10 +19,9 @@ class Tests(tests.IrisTest): # All 'real' tests have been done for load_meshes(). Here we just check # that load_mesh() works with load_meshes() correctly, using mocking. def setUp(self): - patcher = mock.patch("iris.experimental.ugrid.load_meshes") - self.addCleanup(patcher.stop) - - self.load_meshes_mock = patcher.start() + self.load_meshes_mock = self.patch( + "iris.experimental.ugrid.load_meshes" + ) # The expected return from load_meshes - a dict of files, each with # a list of meshes. self.load_meshes_mock.return_value = {"file": ["mesh"]} diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py index d82be68b10..0ec76b5311 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py @@ -16,7 +16,6 @@ from shutil import rmtree from subprocess import check_call import tempfile -from unittest import mock from uuid import uuid4 from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_meshes, logger @@ -193,9 +192,9 @@ class TestsHttp(tests.IrisTest): # Tests of HTTP (OpenDAP) loading need mocking since we can't have tests # that rely on 3rd party servers. def setUp(self): - patcher = mock.patch("iris.fileformats.FORMAT_AGENT.get_spec") - self.addCleanup(patcher.stop) - self.format_agent_mock = patcher.start() + self.format_agent_mock = self.patch( + "iris.fileformats.FORMAT_AGENT.get_spec" + ) def test_http(self): url = "http://foo" From b28fbd80bdb8007dc0c71831cd62e85756c5fc4f Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 13:16:37 +0100 Subject: [PATCH 18/25] Mesh loading clearer docstring. --- lib/iris/experimental/ugrid/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index d50f8d0777..acd6b187f3 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3306,7 +3306,8 @@ def load_mesh(uris, var_name=None): Parameters ---------- uris : str or iterable of str - One or more filenames/URI's. Any URI's must support OpenDAP. + One or more filenames/URI's. Filenames can include wildcards. Any URI's + must support OpenDAP. var_name : str, optional Only return a :class:`Mesh` if its var_name matches this value. @@ -3333,15 +3334,16 @@ def load_meshes(uris, var_name=None): Parameters ---------- uris : str or iterable of str - One or more filenames/URI's. Any URI's must support OpenDAP. + One or more filenames/URI's. Filenames can include wildcards. Any URI's + must support OpenDAP. var_name : str, optional Only return :class:`Mesh`\\ es that have var_names matching this value. Returns ------- dict - A dictionary mapping each file path/URL to a list of the - :class:`Mesh`\\ es returned from each. + A dictionary mapping each mesh-containing file path/URL in the input + ``uris`` to a list of the :class:`Mesh`\\ es returned from each. """ # TODO: rationalise UGRID/mesh handling once experimental.ugrid is folded From bb30d2ec40b7b0912b6b1312bb9491c50923a3d4 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 13:54:50 +0100 Subject: [PATCH 19/25] Enhance test_var_name in test_load_meshes. --- .../experimental/ugrid/test_load_meshes.py | 53 +++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py index 0ec76b5311..30615e0db9 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py @@ -93,6 +93,23 @@ def setUp(self): """ self.nc_path = cdl_to_nc(self.ref_cdl) + def add_second_mesh(self): + second_name = "mesh2" + cdl_extra = f""" + int {second_name} ; + {second_name}:cf_role = "mesh_topology" ; + {second_name}:topology_dimension = 2 ; + {second_name}:node_coordinates = "node_x node_y" ; + {second_name}:face_coordinates = "face_x face_y" ; + {second_name}:face_node_connectivity = "face_nodes" ; + """ + vars_string = "variables:" + vars_start = self.ref_cdl.index(vars_string) + len(vars_string) + new_cdl = ( + self.ref_cdl[:vars_start] + cdl_extra + self.ref_cdl[vars_start:] + ) + return new_cdl, second_name + def test_with_data(self): nc_path = cdl_to_nc(self.ref_cdl) with PARSE_UGRID_ON_LOAD.context(): @@ -123,12 +140,6 @@ def test_no_data(self): mesh = file_meshes[0] self.assertEqual("mesh", mesh.var_name) - def test_var_name(self): - nc_path = cdl_to_nc(self.ref_cdl) - with PARSE_UGRID_ON_LOAD.context(): - meshes = load_meshes(nc_path, "some_other_mesh") - self.assertDictEqual({}, meshes) - def test_multi_files(self): files_count = 3 nc_paths = [cdl_to_nc(self.ref_cdl) for _ in range(files_count)] @@ -137,20 +148,7 @@ def test_multi_files(self): self.assertEqual(files_count, len(meshes)) def test_multi_meshes(self): - cdl_extra = """ - int mesh2 ; - mesh2:cf_role = "mesh_topology" ; - mesh2:topology_dimension = 2 ; - mesh2:node_coordinates = "node_x node_y" ; - mesh2:face_coordinates = "face_x face_y" ; - mesh2:face_node_connectivity = "face_nodes" ; - """ - vars_string = "variables:" - vars_start = self.ref_cdl.index(vars_string) + len(vars_string) - ref_cdl = ( - self.ref_cdl[:vars_start] + cdl_extra + self.ref_cdl[vars_start:] - ) - + ref_cdl, second_name = self.add_second_mesh() nc_path = cdl_to_nc(ref_cdl) with PARSE_UGRID_ON_LOAD.context(): meshes = load_meshes(nc_path) @@ -161,7 +159,20 @@ def test_multi_meshes(self): self.assertEqual(2, len(file_meshes)) mesh_names = [mesh.var_name for mesh in file_meshes] self.assertIn("mesh", mesh_names) - self.assertIn("mesh2", mesh_names) + self.assertIn(second_name, mesh_names) + + def test_var_name(self): + second_cdl, second_name = self.add_second_mesh() + cdls = [self.ref_cdl, second_cdl] + nc_paths = [cdl_to_nc(cdl) for cdl in cdls] + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_paths, second_name) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(1, len(file_meshes)) + self.assertEqual(second_name, file_meshes[0].var_name) def test_no_parsing(self): nc_path = cdl_to_nc(self.ref_cdl) From 2a4237f04cf52472deb976e3e6d4c0aed93362cb Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 14:14:26 +0100 Subject: [PATCH 20/25] Added test_no_mesh to test_load_meshes. --- .../unit/experimental/ugrid/test_load_meshes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py index 30615e0db9..198fc51a2f 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py @@ -140,6 +140,22 @@ def test_no_data(self): mesh = file_meshes[0] self.assertEqual("mesh", mesh.var_name) + def test_no_mesh(self): + cdl_lines = self.ref_cdl.split("\n") + cdl_lines = filter( + lambda line: all( + [s not in line for s in (':mesh = "mesh"', "mesh_topology")] + ), + cdl_lines, + ) + ref_cdl = "\n".join(cdl_lines) + + nc_path = cdl_to_nc(ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path) + + self.assertDictEqual({}, meshes) + def test_multi_files(self): files_count = 3 nc_paths = [cdl_to_nc(self.ref_cdl) for _ in range(files_count)] From ec55e07404debf2623283b28b6591c5bea4ae39c Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 15:30:08 +0100 Subject: [PATCH 21/25] Docstring load clarification for mesh loading. --- lib/iris/experimental/ugrid/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index acd6b187f3..9948c9fc11 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3299,7 +3299,7 @@ def _meshes_from_cf(cf_reader): def load_mesh(uris, var_name=None): """ - Create a single :class:`Mesh` object from one or more NetCDF files. + Load a single :class:`Mesh` object from one or more NetCDF files. Raises an error if more/less than one :class:`Mesh` is found. @@ -3329,7 +3329,7 @@ def load_mesh(uris, var_name=None): def load_meshes(uris, var_name=None): """ - Create :class:`Mesh` objects from one or more NetCDF files. + Load :class:`Mesh` objects from one or more NetCDF files. Parameters ---------- From b900f448ecd3ea1c11659792f896e7484ca47e07 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 16:11:46 +0100 Subject: [PATCH 22/25] load_mesh better duplicate handling. --- lib/iris/experimental/ugrid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 9948c9fc11..027d2d6d68 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3317,7 +3317,7 @@ def load_mesh(uris, var_name=None): """ meshes_result = load_meshes(uris, var_name) - result = [mesh for file in meshes_result.values() for mesh in file] + result = set([mesh for file in meshes_result.values() for mesh in file]) mesh_count = len(result) if mesh_count != 1: message = ( From d8809da8baec16684f98f281574ace3b8bc7438d Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 20 Aug 2021 16:13:54 +0100 Subject: [PATCH 23/25] Removed face coordinates/data from test_load_meshes. --- .../unit/experimental/ugrid/test_load_meshes.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py index 198fc51a2f..8ff0d1ac63 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py @@ -57,16 +57,11 @@ def setUp(self): mesh:cf_role = "mesh_topology" ; mesh:topology_dimension = 2 ; mesh:node_coordinates = "node_x node_y" ; - mesh:face_coordinates = "face_x face_y" ; mesh:face_node_connectivity = "face_nodes" ; float node_x(node) ; node_x:standard_name = "longitude" ; float node_y(node) ; node_y:standard_name = "latitude" ; - float face_x(face) ; - face_x:standard_name = "longitude" ; - float face_y(face) ; - face_y:standard_name = "latitude" ; int face_nodes(face, vertex) ; face_nodes:cf_role = "face_node_connectivity" ; face_nodes:start_index = 0 ; @@ -75,20 +70,13 @@ def setUp(self): node_data:coordinates = "node_x node_y" ; node_data:location = "node" ; node_data:mesh = "mesh" ; - float face_data(levels, face) ; - face_data:coordinates = "face_x face_y" ; - face_data:location = "face" ; - face_data:mesh = "mesh" ; data: mesh = 0; node_x = 0., 2., 1.; node_y = 0., 0., 1.; - face_x = 0.5; - face_y = 0.5; face_nodes = 0, 1, 2; levels = 1, 2; node_data = 0., 0., 0.; - face_data = 0.; } """ self.nc_path = cdl_to_nc(self.ref_cdl) From b16a4cd14d7fa3765c0d31eae64a15f7a15c1835 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Fri, 20 Aug 2021 18:09:42 +0100 Subject: [PATCH 24/25] Allow Meshes to be hashed. --- lib/iris/experimental/ugrid/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 027d2d6d68..2df2db4a41 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -1008,6 +1008,11 @@ def __eq__(self, other): # TBD: this is a minimalist implementation and requires to be revisited return id(self) == id(other) + def __hash__(self): + # Allow use in sets and as dictionary keys, as is done for :class:`iris.cube.Cube`. + # See https://github.com/SciTools/iris/pull/1772 + return hash(id(self)) + def __getstate__(self): return ( self._metadata_manager, From ef1c21702006ecb10e81ed4001682500166990cf Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 23 Aug 2021 09:16:43 +0100 Subject: [PATCH 25/25] Fix for set usage. --- lib/iris/experimental/ugrid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 2df2db4a41..bfc570fcfd 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3329,7 +3329,7 @@ def load_mesh(uris, var_name=None): f"Expecting 1 mesh, but input file(s) produced: {mesh_count} ." ) raise ValueError(message) - return result[0] + return result.pop() # Return the single element def load_meshes(uris, var_name=None):