From 579a44f684d2f6d5e4567688f6f3bec27bb61803 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 16 May 2023 17:26:32 -0400 Subject: [PATCH] Prepare snow depth observations for JEDI-based Land DA. (#1609) This PR: - adds a job to prepare IMS snow depth observations as a task in the workflow. This task depends on the `prep.sh` job to bring IMS data. To test this type of data `DMPDIR` in `config.base` needed to be pointed to `"/scratch1/NCEPDEV/global/Jiarui.Dong/JEDI/GlobalWorkflow/para_gfs/glopara_dump"` - `land_analysis.py` introduces a method `prepare_IMS` for this type of data. - Updates are necessary in the `GDASApp` repo. See companion PR https://github.com/NOAA-EMC/GDASApp/pull/472 This job only runs at the `18z` cycle in the workflow. This is controlled in the XML and not in the script. Consequently, updates are in the `workflow/` scripts to make this happen. Other updates included: - Adds a method (and associated test) to provide a datetime in the form of Julian day of the year. The method is `to_julian` in `timetools.py` - `executable.py` was updated to accept a `list` of arguments (in addition to `str`). **Note:** - `calcfIMS.exe` is a serial executable and yet requires to be run with `srun`. I recommend compiling this executable without MPI. @jiaruidong2017 @CoryMartin-NOAA - There are several deprecation warnings from the python IODA converter that will need to be addressed. I recommend creating an issue to be resolved later. @jiaruidong2017 @CoryMartin-NOAA Part of Land DA work for snow depth assimilation, for which there is no issue open. --- ush/python/pygw/src/pygw/executable.py | 9 ++++--- ush/python/pygw/src/pygw/jinja.py | 3 ++- ush/python/pygw/src/pygw/timetools.py | 29 +++++++++++++++++++-- ush/python/pygw/src/tests/test_timetools.py | 12 ++++++--- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/ush/python/pygw/src/pygw/executable.py b/ush/python/pygw/src/pygw/executable.py index 45464db..e9868b0 100644 --- a/ush/python/pygw/src/pygw/executable.py +++ b/ush/python/pygw/src/pygw/executable.py @@ -2,7 +2,7 @@ import shlex import subprocess import sys -from typing import Any, Optional +from typing import Any, Optional, Union, List __all__ = ["Executable", "which", "CommandNotFoundError"] @@ -47,7 +47,7 @@ def __init__(self, name: str): if not self.exe: raise ProcessError(f"Cannot construct executable for '{name}'") - def add_default_arg(self, arg: str) -> None: + def add_default_arg(self, arg: Union[str, List]) -> None: """ Add a default argument to the command. Parameters @@ -55,7 +55,10 @@ def add_default_arg(self, arg: str) -> None: arg : str argument to the executable """ - self.exe.append(arg) + if isinstance(arg, list): + self.exe.extend(arg) + else: + self.exe.append(arg) def add_default_env(self, key: str, value: Any) -> None: """ diff --git a/ush/python/pygw/src/pygw/jinja.py b/ush/python/pygw/src/pygw/jinja.py index 0e4bcab..56aac05 100644 --- a/ush/python/pygw/src/pygw/jinja.py +++ b/ush/python/pygw/src/pygw/jinja.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Dict -from .timetools import strftime, to_YMDH, to_YMD, to_fv3time, to_isotime +from .timetools import strftime, to_YMDH, to_YMD, to_fv3time, to_isotime, to_julian __all__ = ['Jinja'] @@ -114,6 +114,7 @@ def get_set_env(self, loader: jinja2.BaseLoader) -> jinja2.Environment: env.filters["to_fv3time"] = lambda dt: to_fv3time(dt) if not isinstance(dt, SilentUndefined) else dt env.filters["to_YMDH"] = lambda dt: to_YMDH(dt) if not isinstance(dt, SilentUndefined) else dt env.filters["to_YMD"] = lambda dt: to_YMD(dt) if not isinstance(dt, SilentUndefined) else dt + env.filters["to_julian"] = lambda dt: to_julian(dt) if not isinstance(dt, SilentUndefined) else dt return env @staticmethod diff --git a/ush/python/pygw/src/pygw/timetools.py b/ush/python/pygw/src/pygw/timetools.py index b63e132..cd43b55 100644 --- a/ush/python/pygw/src/pygw/timetools.py +++ b/ush/python/pygw/src/pygw/timetools.py @@ -3,10 +3,10 @@ __all__ = ["to_datetime", "to_timedelta", - "datetime_to_YMDH", "datetime_to_YMD", + "datetime_to_YMDH", "datetime_to_YMD", "datetime_to_JDAY", "timedelta_to_HMS", "strftime", "strptime", - "to_YMDH", "to_YMD", + "to_YMDH", "to_YMD", "to_JDAY", "to_julian", "to_isotime", "to_fv3time", "add_to_datetime", "add_to_timedelta"] @@ -151,6 +151,29 @@ def datetime_to_YMD(dt: datetime.datetime) -> str: raise Exception(f"Bad datetime: '{dt}'") +def datetime_to_JDAY(dt: datetime.datetime) -> str: + """ + Description + ----------- + Translate a datetime object to 'YYYYDOY' format. + + + Parameters + ---------- + dt : datetime.datetime + Datetime object to translate + + Returns + ------- + str: str + Formatted string in 'YYYYDOY' format. + """ + try: + return dt.strftime('%Y%j') + except Exception: + raise Exception(f"Bad datetime: '{dt}'") + + def timedelta_to_HMS(td: datetime.timedelta) -> str: """ Description @@ -289,3 +312,5 @@ def add_to_timedelta(td1, td2): to_YMDH = datetime_to_YMDH to_YMD = datetime_to_YMD +to_JDAY = datetime_to_JDAY +to_julian = datetime_to_JDAY diff --git a/ush/python/pygw/src/tests/test_timetools.py b/ush/python/pygw/src/tests/test_timetools.py index 592fd47..f7e2cfd 100644 --- a/ush/python/pygw/src/tests/test_timetools.py +++ b/ush/python/pygw/src/tests/test_timetools.py @@ -45,7 +45,7 @@ def test_strftime(): def test_strptime(): assert strptime(current_date.strftime('%Y%m%d'), '%Y%m%d') == \ - datetime.strptime(current_date.strftime('%Y%m%d'), '%Y%m%d') + datetime.strptime(current_date.strftime('%Y%m%d'), '%Y%m%d') def test_to_isotime(): @@ -56,13 +56,17 @@ def test_to_fv3time(): assert to_fv3time(current_date) == current_date.strftime('%Y%m%d.%H%M%S') +def test_to_julian(): + assert to_julian(current_date) == current_date.strftime('%Y%j') + + def test_add_to_timedelta(): assert add_to_timedelta(timedelta(days=1), timedelta(hours=3)) == \ - timedelta(days=1, hours=3) + timedelta(days=1, hours=3) assert add_to_timedelta(timedelta(hours=5, minutes=30), timedelta(minutes=15)) == \ - timedelta(hours=5, minutes=45) + timedelta(hours=5, minutes=45) assert add_to_timedelta(timedelta(seconds=45), timedelta(milliseconds=500)) == \ - timedelta(seconds=45, milliseconds=500) + timedelta(seconds=45, milliseconds=500) def test_add_to_datetime():