feat/geozarr model

.pre-commit-config.yaml

@@ -5,13 +5,13 @@ repos:
       - id: validate-pyproject
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 6.0.1
     hooks:
       - id: isort
         language_version: python
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.4
+    rev: v0.13.1
     hooks:
       - id: ruff
         args: ["--fix"]
@@ -24,6 +24,5 @@ repos:
         language_version: python
         exclude: tests/.*
         additional_dependencies:
-        - types-simplejson
         - types-attrs
-        - pydantic~=2.0
+        - pydantic>=2.11

pyproject.toml

@@ -28,6 +28,7 @@ classifiers = [
 requires-python = ">=3.11"
 dependencies = [
     "pydantic-zarr>=0.8.0",
+    "pydantic>=2.12.0a1",
     "zarr>=3.1.1",
     "xarray>=2025.7.1",
     "dask[array,distributed]>=2025.5.1",
@@ -111,7 +112,7 @@ use_parentheses = true
 ensure_newline_before_comments = true
 
 [tool.mypy]
-python_version = "3.10"
+python_version = "3.11"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

src/eopf_geozarr/cli.py

@@ -410,9 +410,9 @@ def render_node(node: Any, path: str = "", level: int = 0) -> str:
         # Generate HTML for this node
         node_html = f"""
         <div class="tree-node" style="margin-left: {level * 20}px;">
-            <details class="tree-details" {'open' if level < 2 else ''}>
+            <details class="tree-details" {"open" if level < 2 else ""}>
                 <summary class="tree-summary">
-                    <span class="tree-icon">{'📁' if children_count > 0 else '📄'}</span>
+                    <span class="tree-icon">{"📁" if children_count > 0 else "📄"}</span>
                     <span class="tree-name">{node_name}</span>
                     <span class="tree-info">({summary})</span>
                 </summary>
@@ -882,7 +882,7 @@ def _generate_html_output(
                 </div>
                 <div class="header-info-item">
                     <div class="header-info-label">Generated</div>
-                    <div class="header-info-value">{__import__('datetime').datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</div>
+                    <div class="header-info-value">{__import__("datetime").datetime.now().strftime("%Y-%m-%d %H:%M:%S")}</div>
                 </div>
             </div>
         </div>

src/eopf_geozarr/data_api/geozarr/common.py

@@ -0,0 +1,264 @@
+"""Common utilities for GeoZarr data API."""
+
+import io
+import urllib
+import urllib.request
+from typing import Annotated, Any, Mapping, TypeVar
+
+from cf_xarray.utils import parse_cf_standard_name_table
+from pydantic import AfterValidator, BaseModel
+from pydantic.experimental.missing_sentinel import MISSING
+from typing_extensions import Protocol, runtime_checkable
+
+from eopf_geozarr.data_api.geozarr.types import ResamplingMethod
+
+
+class BaseDataArrayAttrs(BaseModel, extra="allow"):
+    """
+    Base attributes for a  GeoZarr DataArray.
+
+    Attributes
+    ----------
+    """
+
+    grid_mapping: str | MISSING = MISSING
+
+
+class GridMappingAttrs(BaseModel, extra="allow"):
+    """
+    Grid mapping attributes for a GeoZarr grid mapping variable.
+
+    Attributes
+    ----------
+    grid_mapping_name : str
+        The name of the grid mapping.
+
+    Extra fields are permitted.
+
+    Additional attributes might be present depending on the type of grid mapping.
+
+    References
+    ----------
+    https://cfconventions.org/Data/cf-conventions/cf-conventions-1.12/cf-conventions.html#grid-mappings-and-projections
+    """
+
+    grid_mapping_name: str
+
+
+def get_cf_standard_names(url: str) -> tuple[str, ...]:
+    """Retrieve the set of CF standard names and return them as a tuple."""
+
+    headers = {"User-Agent": "eopf_geozarr"}
+
+    req = urllib.request.Request(url, headers=headers)
+
+    try:
+        with urllib.request.urlopen(req) as response:
+            content = response.read()  # Read the entire response body into memory
+            content_fobj = io.BytesIO(content)
+    except urllib.error.URLError as e:
+        raise e
+
+    _info, table, _aliases = parse_cf_standard_name_table(source=content_fobj)
+    return tuple(table.keys())
+
+
+# This is a URL to the CF standard names table.
+CF_STANDARD_NAME_URL = (
+    "https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/"
+    "master/Data/cf-standard-names/current/src/cf-standard-name-table.xml"
+)
+
+# this does IO against github. consider locally storing this data instead if fetching every time
+# is problematic.
+CF_STANDARD_NAMES = get_cf_standard_names(url=CF_STANDARD_NAME_URL)
+
+
+def check_standard_name(name: str) -> str:
+    """
+    Check if the standard name is valid according to the CF conventions.
+
+    Parameters
+    ----------
+    name : str
+        The standard name to check.
+
+    Returns
+    -------
+    str
+        The validated standard name.
+
+    Raises
+    ------
+    ValueError
+        If the standard name is not valid.
+    """
+
+    if name in CF_STANDARD_NAMES:
+        return name
+    raise ValueError(
+        f"Invalid standard name: {name}. This name was not found in the list of CF standard names."
+    )
+
+
+CFStandardName = Annotated[str, AfterValidator(check_standard_name)]
+
+
+@runtime_checkable
+class GroupLike(Protocol):
+    members: Mapping[str, Any] | None
+    attributes: Any
+
+
+TGroupLike = TypeVar("TGroupLike", bound=GroupLike)
+
+
+def check_valid_coordinates(model: TGroupLike) -> TGroupLike:
+    """
+    Check if the coordinates of the DataArrayLike objects listed in GroupLike objects are valid.
+
+    For each DataArrayLike in the model, we check the dimensions associated with the DataArrayLike.
+    For each dimension associated with a data variable, a DataArrayLike with the name of that data
+    variable must be present in the members of the group.
+
+    Parameters
+    ----------
+    model : GroupLike
+        An object that implements the GroupLike protocol.
+
+    Returns
+    -------
+    GroupLike
+        A GroupLike object with referentially valid coordinates.
+    """
+    if model.members is None:
+        raise ValueError("Model members cannot be None")
+
+    arrays: dict[str, DataArrayLike] = {
+        k: v for k, v in model.members.items() if isinstance(v, DataArrayLike)
+    }
+    for key, array in arrays.items():
+        for idx, dim in enumerate(array.array_dimensions):
+            if dim not in model.members:
+                raise ValueError(
+                    f"Dimension '{dim}' for array '{key}' is not defined in the model members."
+                )
+            member = model.members[dim]
+            if isinstance(member, GroupLike):
+                raise ValueError(
+                    f"Dimension '{dim}' for array '{key}' should be a group. Found an array instead."
+                )
+            if member.shape[0] != array.shape[idx]:
+                raise ValueError(
+                    f"Dimension '{dim}' for array '{key}' has a shape mismatch: "
+                    f"{member.shape[0]} != {array.shape[idx]}."
+                )
+    return model
+
+
+@runtime_checkable
+class DataArrayLike(Protocol):
+    """
+    This is a protocol that models the relevant properties of Zarr V2 and Zarr V3 DataArrays.
+    """
+
+    @property
+    def array_dimensions(self) -> tuple[str, ...]: ...
+
+    shape: tuple[int, ...]
+    attributes: BaseDataArrayAttrs
+
+
+class TileMatrixLimit(BaseModel):
+    """"""
+
+    tileMatrix: str
+    minTileCol: int
+    minTileRow: int
+    maxTileCol: int
+    maxTileRow: int
+
+
+class TileMatrix(BaseModel):
+    id: str
+    scaleDenominator: float
+    cellSize: float
+    pointOfOrigin: tuple[float, float]
+    tileWidth: int
+    tileHeight: int
+    matrixWidth: int
+    matrixHeight: int
+
+
+class TileMatrixSet(BaseModel):
+    id: str
+    title: str | None = None
+    crs: str | None = None
+    supportedCRS: str | None = None
+    orderedAxes: tuple[str, str] | None = None
+    tileMatrices: tuple[TileMatrix, ...]
+
+
+class Multiscales(BaseModel, extra="allow"):
+    """
+    Multiscale metadata for a GeoZarr dataset.
+
+    Attributes
+    ----------
+    tile_matrix_set : str
+        The tile matrix set identifier for the multiscale dataset.
+    resampling_method : ResamplingMethod
+        The name of the resampling method for the multiscale dataset.
+    tile_matrix_set_limits : dict[str, TileMatrixSetLimits] | None, optional
+        The tile matrix set limits for the multiscale dataset.
+    """
+
+    tile_matrix_set: TileMatrixSet
+    resampling_method: ResamplingMethod
+    # TODO: ensure that the keys match tile_matrix_set.tileMatrices[$index].id
+    # TODO: ensure that the keys match the tileMatrix attribute
+    tile_matrix_limits: dict[str, TileMatrixLimit] | None = None
+
+
+class DatasetAttrs(BaseModel, extra="allow"):
+    """
+    Attributes for a GeoZarr dataset.
+
+    A dataset is a collection of DataArrays. This class models the attributes of a dataset
+    """
+
+    ...
+
+
+@runtime_checkable
+class DatasetLike(Protocol):
+    members: Mapping[str, DataArrayLike] | None
+
+
+TDataSetLike = TypeVar("TDataSetLike", bound=DatasetLike)
+
+
+def check_grid_mapping(model: TDataSetLike) -> TDataSetLike:
+    """
+    Ensure that a grid mapping variable is present, and that it refers to a member of the model.
+    """
+    if model.members is not None:
+        for name, member in model.members.items():
+            if member.attributes.grid_mapping not in model.members:
+                msg = f"Grid mapping variable '{member.attributes.grid_mapping}' declared by {name} was not found in dataset members"
+                raise ValueError(msg)
+    return model
+
+
+class MultiscaleGroupAttrs(BaseModel, extra="allow"):
+    """
+    Attributes for Multiscale GeoZarr dataset.
+
+    A Multiscale dataset is a collection of Dataet
+
+    Attributes
+    ----------
+    multiscales: MultiscaleAttrs
+    """
+
+    multiscales: Multiscales