From 5d6c71ab34667ef0dac777a89a19a37edd2652fc Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 2 Mar 2023 11:53:10 -0500 Subject: [PATCH] Add options while setting up Rocoto XML that are useful for CI (#1365) Adds options for maximum tries (default is 2), cyclethrottle (default is 3), taskthrottle (default is 25) and verbosity (default is 10) --- docs/source/setup.rst | 2 ++ workflow/rocoto/workflow_xml.py | 12 +++++++----- workflow/setup_xml.py | 15 ++++++++++++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 19e3f9ac3f..a4e70fbfcb 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -156,6 +156,8 @@ Example: ./setup_xml.py /some_safe_disk_area/Joe.Schmo/expdir/test +Additional options for setting up Rocoto are available with `setup_xml.py -h` that allow users to change the number of failed tries, number of concurrent cycles and tasks as well as Rocoto's verbosity levels. + **************************************** Step 4: Confirm files from setup scripts **************************************** diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py index 4e43ff8ec1..856cade2b9 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/workflow/rocoto/workflow_xml.py @@ -5,6 +5,7 @@ from datetime import datetime from pygw.timetools import to_timedelta from collections import OrderedDict +from typing import Dict from applications import AppConfig from rocoto.workflow_tasks import get_wf_tasks import rocoto.rocoto as rocoto @@ -12,9 +13,10 @@ class RocotoXML: - def __init__(self, app_config: AppConfig) -> None: + def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: self._app_config = app_config + self.rocoto_config = rocoto_config self._base = self._app_config.configs['base'] @@ -60,7 +62,7 @@ def _get_definitions(self) -> str: entity['ROTDIR'] = self._base['ROTDIR'] entity['JOBS_DIR'] = self._base['BASE_JOB'] - entity['MAXTRIES'] = self._base.get('ROCOTO_MAXTRIES', 2) + entity['MAXTRIES'] = self.rocoto_config['maxtries'] # Put them all in an XML key-value syntax strings = [] @@ -75,9 +77,9 @@ def _get_workflow_header(self): """ scheduler = self._app_config.scheduler - cyclethrottle = self._base.get('ROCOTO_CYCLETHROTTLE', 3) - taskthrottle = self._base.get('ROCOTO_TASKTHROTTLE', 25) - verbosity = self._base.get('ROCOTO_VERBOSITY', 10) + cyclethrottle = self.rocoto_config['cyclethrottle'] + taskthrottle = self.rocoto_config['taskthrottle'] + verbosity = self.rocoto_config['verbosity'] expdir = self._base['EXPDIR'] diff --git a/workflow/setup_xml.py b/workflow/setup_xml.py index 09212b59d9..d43efe21e1 100755 --- a/workflow/setup_xml.py +++ b/workflow/setup_xml.py @@ -28,6 +28,15 @@ def input_args(): parser.add_argument('expdir', help='full path to experiment directory containing config files', type=str, default=os.environ['PWD']) + parser.add_argument('--maxtries', help='maximum number of retries', type=int, + default=2, required=False) + parser.add_argument('--cyclethrottle', help='maximum number of concurrent cycles', type=int, + default=3, required=False) + parser.add_argument('--taskthrottle', help='maximum number of concurrent tasks', type=int, + default=25, required=False) + parser.add_argument('--verbosity', help='verbosity level of Rocoto', type=int, + default=10, required=False) + args = parser.parse_args() return args @@ -45,6 +54,10 @@ def check_expdir(cmd_expdir, cfg_expdir): if __name__ == '__main__': user_inputs = input_args() + rocoto_param_dict = {'maxtries': user_inputs.maxtries, + 'cyclethrottle': user_inputs.cyclethrottle, + 'taskthrottle': user_inputs.taskthrottle, + 'verbosity': user_inputs.verbosity} cfg = Configuration(user_inputs.expdir) @@ -54,5 +67,5 @@ def check_expdir(cmd_expdir, cfg_expdir): app_config = AppConfig(cfg) # Create Rocoto Tasks and Assemble them into an XML - xml = RocotoXML(app_config) + xml = RocotoXML(app_config, rocoto_param_dict) xml.write()