Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add NRCan Gridded support #172

Draft
wants to merge 4 commits into
base: fix_eccc
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions miranda/convert/configs/nrcan-gridded-obs_cf_attrs.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
{
"Header": {
"Conventions": "CF-1.9",
"_miranda_version": true,
"_missing_values": [
"-999."
],
"_variable": true,
"acknowledgement": "This data is provided by Natural Resources Canada (NRCan).",
"author": "Natural Resources Canada",
"citation": "When referring to the ANUSPLIN gridded climate data set, the authors provide this data under the Environment and Climate Change Canada Data Server End Use License (<https://open.canada.ca/en/open-government-licence-canada>) and demand that any references to or derivative works based on this data set be cited as follows: McKenney, D.W., Hutchinson, M., Papadopol, P., Lawrence, K., Pedlar, J.H., Campbell, K.L., Milewska, E., Hopkinson, R.F., Price, D., Owen, T., 2011a. Customized spatial climate models for North America. Bulletin of the American Meteorological Society 92(12), pp. 1611-1622. https://doi.org/10.1175/2011BAMS3132.1",
"contact": "dan.mckenney@NRCan-RNCan.gc.ca",
"documentation": "https://www.canada.ca/en/environment-climate-change/services/climate-change/canadian-centre-climate-services/display-download/technical-documentation-adjusted-climate-data.html",
"domain": "CAN",
"frequency": "day",
"institution": "GovCan",
"license": "https://open.canada.ca/en/open-government-licence-canada",
"license_type": "permissive",
"organization": "NRCan",
"processing_level": "raw",
"product": "Natural Resources Canada ANUSPLIN interpolated historical climate model dataset",
"realm": "atmos",
"references:": "McKenney, D.W., M.F. Hutchinson, P. Papadopol, K. Lawrence, J. Pedlar, K. Campbell, E. Milewska, R.F. Hopkinson, D. Price, and T. Owen, 2011: Customized Spatial Climate Models for North America. Bull. Amer. Meteor. Soc., 92, 1611–1622, https://doi.org/10.1175/2011BAMS3132.1",
"source": "nrcan",
"title": "NRCan ANUSPLIN 10-Km Gridded Climate dataset",
"table_date": "2024-03-06",
"table_id": "nrcan-gridded-obs",
"type": "reconstruction"
},
"dimensions:": {
"lat": {
"axis": "Y",
"long_name": "Latitude",
"standard_name": "latitude",
"units": "degrees_north"
},
"long": {
"_cf_dimension_name": "lon",
"axis": "X",
"long_name": "Longitude",
"standard_name": "longitude",
"units": "degrees_east"
},
"time": {
"axis": "T",
"calendar": "standard",
"long_name": "Time",
"standard_name": "time"
}
},
"variables": {
"maxt": {
"_cf_variable_name": "tasmax",
"_invert_sign": false,
"_offset_time": false,
"_transformation": "op + 273.15",
"cell_methods": "time: maximum",
"frequency": "day",
"grid_mapping": "regular_lon_lat",
"long_name": "Daily Maximum Near-Surface Air Temperature",
"original_units": "°C",
"original_field": "Maximum Temperature / températures maximales",
"standard_name": "air_temperature",
"units": "K"
},
"mint": {
"_cf_variable_name": "tasmin",
"_invert_sign": false,
"_offset_time": false,
"_transformation": "op + 273.15",
"cell_methods": "time: minimum",
"frequency": "day",
"grid_mapping": "regular_lon_lat",
"long_name": "Minimum Temperature / températures minimales",
"original_units": "°C",
"original_field": "Min Temp (°C)",
"standard_name": "air_temperature",
"units": "K"
},
"pcp": {
"_cf_variable_name": "pr",
"_invert_sign": false,
"_offset_time": false,
"_transformation": "amount2rate",
"cell_methods": "time: mean",
"frequency": "day",
"grid_mapping": "regular_lon_lat",
"long_name": "Precipitation",
"original_field": "precipitation / précipitations",
"original_units": "mm",
"standard_name": "precipitation_flux",
"units": "kg m-2 s-1"
}
}
}
27 changes: 15 additions & 12 deletions miranda/io/_input.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
from __future__ import annotations

import logging.config
import os
from pathlib import Path
import pathlib
from types import GeneratorType

import netCDF4 as nc # noqa
Expand All @@ -20,10 +19,10 @@

# FIXME: How are these two functions different?
def discover_data(
input_files: str | os.PathLike | list[str | os.PathLike] | GeneratorType,
input_files: str | pathlib.Path | list[str | pathlib.Path] | GeneratorType,
suffix: str = "nc",
recurse: bool = True,
) -> list[Path] | GeneratorType:
) -> list[pathlib.Path] | GeneratorType:
"""Discover data.

Parameters
Expand All @@ -43,8 +42,8 @@ def discover_data(
--------
Recursion through ".zarr" files is explicitly disabled. Recursive globs and generators will not be expanded/sorted.
"""
if isinstance(input_files, (Path, str)):
input_files = Path(input_files)
if isinstance(input_files, (pathlib.Path, str)):
input_files = pathlib.Path(input_files)
if input_files.is_dir():
if suffix.endswith("zarr") or not recurse:
input_files = sorted(list(input_files.glob(f"*.{suffix}")))
Expand All @@ -56,7 +55,7 @@ def discover_data(
)
input_files = [input_files]
elif isinstance(input_files, list):
input_files = sorted(Path(p) for p in input_files)
input_files = sorted(pathlib.Path(p) for p in input_files)
elif isinstance(input_files, GeneratorType):
logging.warning(
"A Generator was passed to `discover_data`. Passing object along..."
Expand All @@ -68,11 +67,11 @@ def discover_data(


def find_filepaths(
source: str | Path | GeneratorType | list[Path | str],
source: str | pathlib.Path | GeneratorType | list[pathlib.Path | str],
recursive: bool = True,
file_suffixes: str | list[str] | None = None,
**_,
) -> list[Path]:
) -> list[pathlib.Path]:
"""Find all available filepaths at a given source.

Parameters
Expand All @@ -91,17 +90,21 @@ def find_filepaths(
file_suffixes = [file_suffixes]

found = list()
if isinstance(source, (Path, str)):
if isinstance(source, (pathlib.Path, str)):
source = [source]

for location in source:
for pattern in file_suffixes:
if "*" not in pattern:
pattern = f"*{pattern}*"
if recursive:
found.extend([f for f in Path(location).expanduser().rglob(pattern)])
found.extend(
[f for f in pathlib.Path(location).expanduser().rglob(pattern)]
)
elif not recursive:
found.extend([f for f in Path(location).expanduser().glob(pattern)])
found.extend(
[f for f in pathlib.Path(location).expanduser().glob(pattern)]
)
else:
raise ValueError(f"Recursive: {recursive}")

Expand Down