diff --git a/.zenodo.json b/.zenodo.json
index bb00d5471d..f0997c6467 100644
--- a/.zenodo.json
+++ b/.zenodo.json
@@ -204,6 +204,11 @@
{
"affiliation": "DLR, Germany",
"name": "Cammarano, Diego"
+ },
+ {
+ "affiliation": "ACCESS-NRI, Australia",
+ "name": "Yousong, Zeng",
+ "orcid": "0000-0002-8385-5367"
}
],
"description": "ESMValCore: A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts.",
diff --git a/CITATION.cff b/CITATION.cff
index 562e044ecb..0ab14bf848 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -209,6 +209,11 @@ authors:
family-names: Cammarano
given-names: Diego
+ affiliation: "ACCESS-NRI, Australia"
+ family-names: Yousong
+ given-names: Zeng
+ orcid: "https://orcid.org/0000-0002-8385-5367"
+
cff-version: 1.2.0
date-released: 2024-05-08
doi: "10.5281/zenodo.3387139"
diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst
index 5692a9f0b1..1c69acb6f7 100644
--- a/doc/quickstart/find_data.rst
+++ b/doc/quickstart/find_data.rst
@@ -564,6 +564,75 @@ explained in :ref:`extra_facets`, and which content is :download:`available here
`. These multi-variable
files must also undergo some data selection.
+.. _read_access-esm:
+
+ACCESS-ESM
+^^^^^^^^^^
+
+ESMValTool can read native `ACCESS-ESM `__
+model output.
+
+.. warning::
+
+ This is the first version of ACCESS-ESM CMORizer for ESMValCore Currently,
+ Supported variables: ``pr``, ``ps``, ``psl``, ``rlds``, ``tas``, ``ta``, ``va``,
+ ``ua``, ``zg``, ``hus``, ``clt``, ``rsus``, ``rlus``.
+
+This is an example of the directory file structure in `config_developer.yml`.
+
+.. code-block:: yaml
+
+ ACCESS:
+ cmor_strict: false
+ input_dir:
+ default:
+ - '{institute}/{sub_dataset}/{exp}/{modeling_realm}/netCDF'
+ input_file:
+ default: '{sub_dataset}.{special_attr}-*.nc'
+ output_file: '{institute}_{sub_dataset}_{special_attr}_{short_name}'
+ cmor_type: 'CMIP6'
+ cmor_default_table_prefix: 'CMIP6_'
+
+.. hint::
+
+ We only provide one default `input_dir` since this is how ACCESS-ESM native data was
+ stored on NCI. Users can modify this path to match their local file structure.
+
+
+Thus, example dataset entries could look like this:
+
+.. code-block:: yaml
+
+ dataset:
+ - {project: ACCESS, institute: ACCESS-ESM1-5, mip: Amon, dataset:ACCESS_ESM, sub_dataset: HI-CN-05,
+ exp: history, modeling_realm: atm, special_attr: pa, start_year: 1986, end_year: 1986}
+
+
+`dataset` and `sub_dataset` are not redundant, `dataset` is for ESMValCore to search for CMORizer,
+which is always `ACCESS_ESM`, `sub_dataset` is dataset under root `ACCESS-ESM1-5`.
+
+`special_attr` is a special attribute in the filename `ACCESS-ESM` raw data, it's related to frquency
+of raw data.
+
+`modeling_realm` is a realm attribute, it include `atm`, `ice` and `oce`.
+
+Similar to any other fix, the ACCESS-ESM fix allows the use of :ref:`extra
+facets`.
+By default, the file :download:`emac-mappings.yml
+` is used for that
+purpose.
+For some variables, extra facets are necessary; otherwise ESMValCore cannot
+read them properly.
+Supported keys for extra facets are:
+
+==================== ====================================== =================================
+Key Description Default value if not specified
+==================== ====================================== =================================
+``raw_name`` Variable name of the variable in the CMOR variable name of the
+ raw input file corresponding variable
+``modeling_realm`` Realm attribute include `atm`, `ice` Default realm of this variable
+ and `oce`
+==================== ====================================== =================================
.. _data-retrieval:
diff --git a/esmvalcore/cmor/_fixes/access/__init__.py b/esmvalcore/cmor/_fixes/access/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/esmvalcore/cmor/_fixes/access/access_esm.py b/esmvalcore/cmor/_fixes/access/access_esm.py
new file mode 100644
index 0000000000..6af813ff38
--- /dev/null
+++ b/esmvalcore/cmor/_fixes/access/access_esm.py
@@ -0,0 +1,84 @@
+"""On-the-fly CMORizer for ACCESS-ESM.
+
+Note
+----
+This is the first version of ACCESS-ESM CMORizer in for ESMValCore
+Currently, only two variables (`tas`,`pr`) is fully supported.
+"""
+import logging
+
+from iris.cube import CubeList
+
+from esmvalcore.cmor._fixes.native_datasets import NativeDatasetFix
+
+logger = logging.getLogger(__name__)
+
+
+class AllVars(NativeDatasetFix):
+ """Fixes for all variables."""
+
+ def fix_coord_system(self, cube):
+ """Delete coord_system to make CubeList able to merge."""
+ for dim in cube.dim_coords:
+ if dim.coord_system is not None:
+ cube.coord(dim.standard_name).coord_system = None
+
+ def fix_height_value(self, cube):
+ """Fix height value to make it comparable to other dataset."""
+ if cube.coord('height').points[0] != 2:
+ cube.coord('height').points = [2]
+
+ def get_cube_from_multivar(self, cubes):
+ """Get cube before calculate from multiple variables."""
+ rawname_list = self.extra_facets.get('raw_name',
+ self.vardef.short_name)
+ calculate = self.extra_facets.get('calculate', self.vardef.short_name)
+ var = []
+ for rawname in rawname_list:
+ var.append(self.get_cube(cubes, rawname))
+ return eval(calculate)
+
+ def fix_metadata(self, cubes):
+ """Fix metadata.
+
+ Fix name of coordinate(height), long name and variable name of
+ variable(tas).
+
+ Parameters
+ ----------
+ cubes : iris.cube.CubeList
+ Input cubes.
+
+ Returns
+ -------
+ iris.cube.CubeList
+ """
+ if isinstance(
+ self.extra_facets.get('raw_name', self.vardef.short_name),
+ list):
+ cube = self.get_cube_from_multivar(cubes)
+ else:
+ cube = self.get_cube(cubes)
+
+ # Fix scalar coordinates
+ self.fix_scalar_coords(cube)
+
+ # Fix metadata of variable
+ self.fix_var_metadata(cube)
+
+ # Fix metadata coordinates
+ self.fix_lon_metadata(cube)
+ self.fix_lat_metadata(cube)
+
+ # Fix coordinate 'height'
+ if 'height_0' in [var.var_name for var in cube.coords()]:
+ self.fix_height_metadata(cube)
+ self.fix_height_value(cube)
+ # Fix coordinate 'pressure'
+ if 'pressure' in [var.var_name for var in cube.coords()]:
+ self.fix_plev_metadata(cube, coord='pressure')
+
+ # Fix coord system
+ self.fix_coord_system(cube)
+
+ return CubeList([cube])
diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml
index 9d5f1bc62f..090abfed8b 100644
--- a/esmvalcore/config-developer.yml
+++ b/esmvalcore/config-developer.yml
@@ -194,3 +194,14 @@ CESM:
output_file: '{project}_{dataset}_{case}_{gcomp}_{scomp}_{type}_{mip}_{short_name}'
cmor_type: 'CMIP6'
cmor_default_table_prefix: 'CMIP6_'
+
+ACCESS:
+ cmor_strict: false
+ input_dir:
+ default:
+ - '{institute}/{sub_dataset}/{exp}/{modeling_realm}/netCDF'
+ input_file:
+ default: '{sub_dataset}.{special_attr}-*.nc'
+ output_file: '{institute}_{sub_dataset}_{special_attr}_{short_name}'
+ cmor_type: 'CMIP6'
+ cmor_default_table_prefix: 'CMIP6_'
diff --git a/esmvalcore/config/extra_facets/access-mappings.yml b/esmvalcore/config/extra_facets/access-mappings.yml
new file mode 100644
index 0000000000..44da01d7fd
--- /dev/null
+++ b/esmvalcore/config/extra_facets/access-mappings.yml
@@ -0,0 +1,70 @@
+# Extra facets for native ACCESS model output
+
+# A complete list of supported keys is given in the documentation (see
+# ESMValCore/doc/quickstart/find_data.rst).
+---
+
+ACCESS_ESM:
+
+ '*':
+
+ tas:
+ raw_name: fld_s03i236
+ modeling_realm: atm
+
+ pr:
+ raw_name: fld_s05i216
+ modeling_realm: atm
+
+ ps:
+ raw_name: fld_s00i409
+ modeling_realm: atm
+
+ clt:
+ raw_name: fld_s02i204
+ modeling_realm: atm
+
+ psl:
+ raw_name: fld_s16i222
+ modeling_realm: atm
+
+ hus:
+ raw_name: fld_s30i205
+ modeling_realm: atm
+
+ zg:
+ raw_name: fld_s30i207
+ modeling_realm: atm
+
+ va:
+ raw_name: fld_s30i202
+ modeling_realm: atm
+
+ ua:
+ raw_name: fld_s30i201
+ modeling_realm: atm
+
+ ta:
+ raw_name: fld_s30i204
+ modeling_realm: atm
+
+ rlus:
+ raw_name:
+ - fld_s02i207
+ - fld_s02i201
+ - fld_s03i332
+ - fld_s02i205
+ modeling_realm: atm
+ calculate: var[0]-var[1]+var[2]-var[3]
+
+ rlds:
+ raw_name: fld_s02i207
+ modeling_realm: atm
+
+ rsus:
+ raw_name:
+ - fld_s01i235
+ - fld_s01i201
+ modeling_realm: atm
+ calculate: var[0]-var[1]
+
diff --git a/tests/integration/cmor/_fixes/access/__init__.py b/tests/integration/cmor/_fixes/access/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/integration/cmor/_fixes/access/test_access_esm.py b/tests/integration/cmor/_fixes/access/test_access_esm.py
new file mode 100644
index 0000000000..e44863ac4b
--- /dev/null
+++ b/tests/integration/cmor/_fixes/access/test_access_esm.py
@@ -0,0 +1,288 @@
+"""Tests for the ACCESS-ESM on-the-fly CMORizer."""
+
+import dask.array as da
+import iris
+import numpy as np
+import pytest
+
+import esmvalcore.cmor._fixes.access.access_esm
+from esmvalcore.cmor._fixes.fix import GenericFix
+from esmvalcore.cmor.fix import Fix
+from esmvalcore.cmor.table import CoordinateInfo, get_var_info
+from esmvalcore.config._config import get_extra_facets
+from esmvalcore.dataset import Dataset
+from iris.coords import DimCoord
+from iris.cube import Cube, CubeList
+from cf_units import Unit
+
+
+@pytest.fixture
+def cubes_2d(test_data_path):
+ """2D sample cubes."""
+ nc_path = test_data_path / 'access_native.nc'
+ return iris.load(str(nc_path))
+
+
+def _get_fix(mip, frequency, short_name, fix_name):
+ """Load a fix from :mod:`esmvalcore.cmor._fixes.cesm.cesm2`."""
+ dataset = Dataset(
+ project='ACCESS',
+ dataset='ACCESS_ESM',
+ mip=mip,
+ short_name=short_name,
+ )
+ extra_facets = get_extra_facets(dataset, ())
+ extra_facets['frequency'] = frequency
+ extra_facets['exp'] = 'amip'
+ vardef = get_var_info(project='ACCESS', mip=mip, short_name=short_name)
+ cls = getattr(esmvalcore.cmor._fixes.access.access_esm, fix_name)
+ fix = cls(vardef, extra_facets=extra_facets, session={}, frequency='')
+ return fix
+
+
+def get_fix(mip, frequency, short_name):
+ """Load a variable fix from esmvalcore.cmor._fixes.cesm.cesm."""
+ fix_name = short_name[0].upper() + short_name[1:]
+ return _get_fix(mip, frequency, short_name, fix_name)
+
+
+def get_fix_allvar(mip, frequency, short_name):
+ """Load a variable fix from esmvalcore.cmor._fixes.cesm.cesm."""
+ return _get_fix(mip, frequency, short_name, 'AllVars')
+
+
+def fix_metadata(cubes, mip, frequency, short_name):
+ """Fix metadata of cubes."""
+ fix = get_fix(mip, frequency, short_name)
+ cubes = fix.fix_metadata(cubes)
+ return cubes
+
+
+def check_tas_metadata(cubes):
+ """Check tas metadata."""
+ assert len(cubes) == 1
+ cube = cubes[0]
+ assert cube.var_name == 'tas'
+ assert cube.standard_name == 'air_temperature'
+ assert cube.long_name == 'Near-Surface Air Temperature'
+ assert cube.units == 'K'
+ assert 'positive' not in cube.attributes
+ return cube
+
+
+def check_hus_metadata(cubes):
+ """Check hus metadata."""
+ assert len(cubes) == 1
+ cube = cubes[0]
+ assert cube.var_name == 'hus'
+ assert cube.standard_name == 'precipitation_flux'
+ assert cube.long_name == 'Precipitation'
+ assert cube.units == 'kg m-2 s-1'
+ assert 'positive' not in cube.attributes
+ return cube
+
+
+def check_time(cube):
+ """Check time coordinate of cube."""
+ assert cube.coords('time', dim_coords=True)
+ time = cube.coord('time', dim_coords=True)
+ assert time.var_name == 'time'
+ assert time.standard_name == 'time'
+ assert time.bounds.shape == (1, 2)
+ assert time.attributes == {}
+
+
+def check_lat(cube):
+ """Check latitude coordinate of cube."""
+ assert cube.coords('latitude', dim_coords=True)
+ lat = cube.coord('latitude', dim_coords=True)
+ assert lat.var_name == 'lat'
+ assert lat.standard_name == 'latitude'
+ assert lat.units == 'degrees_north'
+ assert lat.attributes == {}
+
+
+def check_lon(cube):
+ """Check longitude coordinate of cube."""
+ assert cube.coords('longitude', dim_coords=True)
+ lon = cube.coord('longitude', dim_coords=True)
+ assert lon.var_name == 'lon'
+ assert lon.standard_name == 'longitude'
+ assert lon.units == 'degrees_east'
+ assert lon.attributes == {}
+
+
+def check_heightxm(cube, height_value):
+ """Check scalar heightxm coordinate of cube."""
+ assert cube.coords('height')
+ height = cube.coord('height')
+ assert height.var_name == 'height'
+ assert height.standard_name == 'height'
+ assert height.units == 'm'
+ assert height.attributes == {'positive': 'up'}
+ np.testing.assert_allclose(height.points, [height_value])
+ assert height.bounds is None
+
+
+def assert_plev_metadata(self, cube):
+ """Assert plev metadata is correct."""
+ assert cube.coord('air_pressure').standard_name == 'air_pressure'
+ assert cube.coord('air_pressure').var_name == 'plev'
+ assert cube.coord('air_pressure').units == 'Pa'
+ assert cube.coord('air_pressure').attributes == {}
+
+
+def test_only_time(monkeypatch, cubes_2d):
+ """Test fix."""
+ fix = get_fix_allvar('Amon', 'mon', 'tas')
+
+ coord_info = CoordinateInfo('time')
+ coord_info.standard_name = 'time'
+ monkeypatch.setattr(fix.vardef, 'coordinates', {'time': coord_info})
+
+ cubes = cubes_2d
+ fixed_cubes = fix.fix_metadata(cubes)
+
+ # Check cube metadata
+ cube = check_tas_metadata(fixed_cubes)
+
+ # Check cube data
+ assert cube.shape == (1, 145, 192)
+
+ # Check time metadata
+ assert cube.coords('time')
+ new_time_coord = cube.coord('time', dim_coords=True)
+ assert new_time_coord.var_name == 'time'
+ assert new_time_coord.standard_name == 'time'
+
+
+def test_only_latitude(monkeypatch, cubes_2d):
+ """Test fix."""
+ fix = get_fix_allvar('Amon', 'mon', 'tas')
+
+ coord_info = CoordinateInfo('latitude')
+ coord_info.standard_name = 'latitude'
+ monkeypatch.setattr(fix.vardef, 'coordinates', {'latitude': coord_info})
+
+ cubes = cubes_2d
+ fixed_cubes = fix.fix_metadata(cubes)
+
+ # Check cube metadata
+ cube = check_tas_metadata(fixed_cubes)
+
+ # Check cube data
+ assert cube.shape == (1, 145, 192)
+
+ # Check latitude metadata
+ assert cube.coords('latitude', dim_coords=True)
+ new_lat_coord = cube.coord('latitude')
+ assert new_lat_coord.var_name == 'lat'
+ assert new_lat_coord.standard_name == 'latitude'
+ assert new_lat_coord.units == 'degrees_north'
+
+
+def test_only_longitude(monkeypatch, cubes_2d):
+ """Test fix."""
+ fix = get_fix_allvar('Amon', 'mon', 'tas')
+
+ coord_info = CoordinateInfo('longitude')
+ coord_info.standard_name = 'longitude'
+ monkeypatch.setattr(fix.vardef, 'coordinates', {'longitude': coord_info})
+
+ cubes = cubes_2d
+ fixed_cubes = fix.fix_metadata(cubes)
+
+ # Check cube metadata
+ cube = check_tas_metadata(fixed_cubes)
+
+ # Check cube data
+ assert cube.shape == (1, 145, 192)
+
+ # Check longitude metadata
+ assert cube.coords('longitude', dim_coords=True)
+ new_lon_coord = cube.coord('longitude')
+ assert new_lon_coord.var_name == 'lon'
+ assert new_lon_coord.standard_name == 'longitude'
+ assert new_lon_coord.units == 'degrees_east'
+
+
+def test_get_tas_fix():
+ """Test getting of fix 'tas'."""
+ fix = Fix.get_fixes('ACCESS', 'ACCESS_ESM', 'Amon', 'tas')
+ assert fix == [
+ esmvalcore.cmor._fixes.access.access_esm.AllVars(vardef={},
+ extra_facets={},
+ session={},
+ frequency=''),
+ GenericFix(None),
+ ]
+
+
+def test_tas_fix(cubes_2d):
+ """Test fix 'tas'."""
+ fix = get_fix_allvar('Amon', 'mon', 'tas')
+ fixed_cubes = fix.fix_metadata(cubes_2d)
+ fixed_cube = check_tas_metadata(fixed_cubes)
+
+ check_time(fixed_cube)
+ check_lat(fixed_cube)
+ check_lon(fixed_cube)
+ check_heightxm(fixed_cube, 2)
+
+ assert fixed_cube.shape == (1, 145, 192)
+
+
+def test_hus_fix():
+ """Test fix 'hus'."""
+ time_coord = DimCoord(
+ [15, 45],
+ standard_name='time',
+ var_name='time',
+ units=Unit('days since 1851-01-01', calendar='noleap'),
+ attributes={'test': 1, 'time_origin': 'will_be_removed'},
+ )
+ plev_coord_rev = DimCoord(
+ [250, 500, 850],
+ standard_name='air_pressure',
+ var_name='plev',
+ units='hPa',
+ )
+ lat_coord_rev = DimCoord(
+ [10, -10],
+ standard_name='latitude',
+ var_name='lat',
+ units='degrees',
+ )
+ lon_coord = DimCoord(
+ [-180, 0],
+ standard_name='longitude',
+ var_name='lon',
+ units='degrees',
+ )
+ coord_spec_4d = [
+ (time_coord, 0),
+ (plev_coord_rev, 1),
+ (lat_coord_rev, 2),
+ (lon_coord, 3),
+ ]
+ cube_4d = Cube(
+ da.arange(2 * 3 * 2 * 2, dtype=np.float32).reshape(2, 3, 2, 2),
+ standard_name='air_pressure',
+ long_name='Air Pressure',
+ var_name='hus',
+ units='celsius',
+ dim_coords_and_dims=coord_spec_4d,
+ attributes={},
+ )
+ cubes_4d = CubeList([cube_4d])
+
+ fix = get_fix_allvar('Amon', 'mon', 'hus')
+ fixed_cubes = fix.fix_metadata(cubes_4d)
+ fixed_cube = check_tas_metadata(fixed_cubes)
+
+ check_time(fixed_cube)
+ check_lat(fixed_cube)
+ check_lon(fixed_cube)
+ assert_plev_metadata(fixed_cube)
+
+ assert fixed_cube.shape == (1, 145, 192)
diff --git a/tests/integration/cmor/_fixes/test_data/access_native.nc b/tests/integration/cmor/_fixes/test_data/access_native.nc
new file mode 100644
index 0000000000..7c0849db63
Binary files /dev/null and b/tests/integration/cmor/_fixes/test_data/access_native.nc differ