Extension of NASA MERRA2 CMORizer (cl, cli, clivi, clw, clwvi) (#3167)

doc/sphinx/source/input.rst

@@ -358,7 +358,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 | MERRA2                       | sm (Lmon)                                                                                            |   3  | Python          |
 |                              | clt, pr, evspsbl, hfss, hfls, huss, prc, prsn, prw, ps, psl, rlds, rldscs, rlus, rlut, rlutcs, rsds, |      |                 |
 |                              | rsdscs, rsdt, tas, tasmin, tasmax, tauu, tauv, ts, uas, vas, rsus, rsuscs, rsut, rsutcs, ta, ua, va, |      |                 |
-|                              | tro3, zg, hus, wap, hur (Amon)                                                                       |      |                 |
+|                              | tro3, zg, hus, wap, hur, cl, clw, cli, clwvi, clivi (Amon)                                           |      |                 |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | MLS-AURA                     | hur, hurStderr (day)                                                                                 |   3  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+

esmvaltool/cmorizers/data/cmor_config/MERRA2.yml

@@ -216,3 +216,28 @@ variables:
     mip: Amon
     raw: "RH"
     file: 'MERRA2_???.instM_3d_asm_Np.{year}??.nc4'
+  cli_monthly:
+    short_name: cli
+    mip: Amon
+    raw: "QI"
+    file: 'MERRA2_???.tavgM_3d_cld_Np.{year}??.nc4'
+  clw_monthly:
+    short_name: clw
+    mip: Amon
+    raw: "QL"
+    file: 'MERRA2_???.tavgM_3d_cld_Np.{year}??.nc4'
+  cl_monthly:
+    short_name: cl
+    mip: Amon
+    raw: "CLOUD"
+    file: 'MERRA2_???.tavgM_3d_cld_Np.{year}??.nc4'
+  clivi_monthly:
+    short_name: clivi
+    mip: Amon
+    raw: TQI
+    file: 'MERRA2_???.instM_2d_asm_Nx.{year}??.nc4'
+  clwvi_monthly:
+    short_name: clwvi
+    mip: Amon
+    raw: TQI+TQL
+    file: 'MERRA2_???.instM_2d_asm_Nx.{year}??.nc4'

esmvaltool/cmorizers/data/downloaders/datasets/merra2.py

@@ -29,7 +29,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date,
     if not start_date:
         start_date = datetime(1980, 1, 1)
     if not end_date:
-        end_date = datetime(2021, 1, 1)
+        end_date = datetime(2022, 1, 1)
     loop_date = start_date
 
     downloader = NASADownloader(
@@ -62,4 +62,10 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date,
         downloader.download_folder(
             "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/"
             f"M2TMNXFLX.5.12.4/{year}/")
+        downloader.download_folder(
+            "https://goldsmr5.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/"
+            f"M2TMNPCLD.5.12.4/{year}/")
+        downloader.download_folder(
+            "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/"
+            f"M2IMNXASM.5.12.4/{year}/")
         loop_date += relativedelta.relativedelta(years=1)

esmvaltool/cmorizers/data/formatters/datasets/merra2.py

@@ -74,6 +74,11 @@ def _var_pairs(cube_list, var_parts, oper):
             cube_1 - cube_2 for cube_1, cube_2 in zip(selected_1, selected_2)
         ]
         selected = iris.cube.CubeList(selected)
+    elif oper == "+":
+        selected = [
+            cube_1 + cube_2 for cube_1, cube_2 in zip(selected_1, selected_2)
+        ]
+        selected = iris.cube.CubeList(selected)
     else:
         raise NotImplementedError(f"Pairwise variables operation {oper} "
                                   "not implemented yet, you can do it "
@@ -156,13 +161,14 @@ def _fix_coordinates(cube, definition):
             if len(coord.points) > 1:
                 coord.guess_bounds()
         else:
-            # special case for UV
-            # variable "uv" (raw: "V") comes with "alevel" instead
+            # special case for UV and 3-dim cloud variables:
+            # variables come with "alevel" instead
             # of "plev19" in the table; "alevel" has empty fields for
             # standard_name, out_name etc. so we need to set them; it's safe
             # to do so since the cmor checker/fixer will convert that during
             # preprocessing at cmor fix stage
-            if cube.var_name == "uv" and axis == "Z":
+            specialvars = ('uv', 'cl', 'cli', 'clw')
+            if cube.var_name in specialvars and axis == "Z":
                 coord = cube.coord(axis=axis)
                 coord_def = definition.coordinates.get('alevel')
                 coord.standard_name = "air_pressure"
@@ -172,6 +178,16 @@ def _fix_coordinates(cube, definition):
                 if len(coord.points) > 1:
                     coord.guess_bounds()
 
+                if coord.units == "hPa":
+                    coord.convert_units('Pa')
+                else:
+                    try:
+                        coord.convert_units('Pa')
+                    except ValueError as exc:
+                        logger.error("Attempting to convert units for "
+                                     "coordinate %s to Pa", coord)
+                        raise exc
+
     return cube
 
 

esmvaltool/recipes/examples/recipe_check_obs.yml

@@ -1719,6 +1719,21 @@ diagnostics:
       hur_monthly:
         short_name: hur
         mip: Amon
+      cl_monthly:
+        short_name: cl
+        mip: Amon
+      clw_monthly:
+        short_name: clw
+        mip: Amon
+      cli_monthly:
+        short_name: cli
+        mip: Amon
+      clwvi_monthly:
+        short_name: clwvi
+        mip: Amon
+      clivi_monthly:
+        short_name: clivi
+        mip: Amon
     additional_datasets:
       - {dataset: MERRA2, project: OBS6, tier: 3,
          type: reanaly, version: 5.12.4, start_year: 1980, end_year: 2022}

tests/unit/cmorizers/obs/test_merra2.py

@@ -176,7 +176,7 @@ def test_load_cube_pairwise_vars_wrong_oper(tmp_path):
     iris.save(cubes, str(path_cubes))
     var = {
         'short_name': 'rsut',
-        'mip': 'Amon', 'raw': 'SWTDN+SWTNT',
+        'mip': 'Amon', 'raw': 'SWTDN:SWTNT',
         'file': 'MERRA2_???.tavgM_2d_rad_Nx.{year}??.nc4'
     }
     in_files = str(tmp_path / "cubes.nc")
@@ -270,7 +270,11 @@ def test_vertical_levels(tmp_path):
     cube_4.var_name = "H"
     cube_4.units = Unit('m')
     cube_4.coord("vertical level").units = "m"
-    cubes = iris.cube.CubeList([cube_1, cube_2, cube_3, cube_4])
+    cube_5 = _create_sample_cube()
+    cube_5.var_name = "QI"
+    cube_5.units = Unit('1')
+    cube_5.coord("vertical level").units = "hPa"
+    cubes = iris.cube.CubeList([cube_1, cube_2, cube_3, cube_4, cube_5])
     iris.save(cubes, str(path_cubes))
     var_1 = {
         'short_name': 'va',
@@ -292,6 +296,11 @@ def test_vertical_levels(tmp_path):
         'mip': 'Amon', 'raw': 'H',
         'file': 'MERRA2_???.instM_3d_ana_Np.{year}??.nc4'
     }
+    var_5 = {
+        'short_name': 'cli',
+        'mip': 'Amon', 'raw': 'QI',
+        'file': 'MERRA2_???.tavgM_3d_cld_Np.{year}??.nc4'
+    }
     in_files = str(tmp_path / "cubes.nc")
     cfg = read_cmor_config("MERRA2")
 
@@ -336,3 +345,10 @@ def test_vertical_levels(tmp_path):
         _extract_variable(in_files, var_4, cfg, tmp_path)
     expected_exc = "Unable to convert from 'Unit('m')' to 'Unit('Pa')'"
     assert expected_exc in str(exc)
+
+    # test unit of vertical coordinate of 3-dim cloud variable
+    _extract_variable(in_files, var_5, cfg, tmp_path)
+    cmorized_data = \
+        tmp_path / "OBS6_MERRA2_reanaly_5.12.4_Amon_cli_198201-198201.nc"
+    cmorized_cube = iris.load_cube(str(cmorized_data))
+    np.testing.assert_equal(cmorized_cube.coord(axis='Z').units, Unit('Pa'))