From 62068fcdb11ce15004dcdfe99579d8dd5c5c99e0 Mon Sep 17 00:00:00 2001 From: James Eddy Date: Fri, 31 Aug 2018 08:50:06 -0700 Subject: [PATCH 1/5] chore: simplify config.yaml and remove from version control Keep only a single queue, tool registry, and workfow service in the app config (config.yaml) to represent "default" options and provide basic examples. Add config file to gitignore so that local changes are no longer tracked. --- .gitignore | 1 + synorchestrator/config.yaml | 24 ------------------------ 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 6d9997a..31cd58e 100644 --- a/.gitignore +++ b/.gitignore @@ -112,5 +112,6 @@ ENV/ # local config files *.config *.evals +*config.yaml .DS_Store diff --git a/synorchestrator/config.yaml b/synorchestrator/config.yaml index 6d66b1f..312ea7e 100644 --- a/synorchestrator/config.yaml +++ b/synorchestrator/config.yaml @@ -11,30 +11,6 @@ queues: workflow_id: '' workflow_type: CWL workflow_url: file://tests/testdata/md5sum.cwl - queue_2: - trs_id: dockstore - version_id: develop - wes_default: local - wes_opts: - - local - workflow_attachments: - - !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/md5sum/md5sum-tool.cwl' - workflow_id: github.com/dockstore-testing/md5sum-checker - workflow_type: CWL - workflow_url: !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/md5sum/md5sum-workflow.cwl' - queue_2_checker: - trs_id: dockstore - version_id: develop - wes_default: local - wes_opts: - - local - workflow_attachments: - - !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/checker/md5sum-checker.cwl' - - !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/md5sum/md5sum-tool.cwl' - - !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/md5sum/md5sum-workflow.cwl' - workflow_id: !!python/unicode 'github.com/dockstore-testing/md5sum-checker/_cwl_checker' - workflow_type: CWL - workflow_url: !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/checker-workflow-wrapping-workflow.cwl' toolregistries: dockstore: auth: '' From 8d0ad035be41c4880ea4a36cea049e1491a946e6 Mon Sep 17 00:00:00 2001 From: James Eddy Date: Fri, 31 Aug 2018 09:29:07 -0700 Subject: [PATCH 2/5] change empty strings in config to null --- synorchestrator/config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synorchestrator/config.yaml b/synorchestrator/config.yaml index 312ea7e..3dd3edd 100644 --- a/synorchestrator/config.yaml +++ b/synorchestrator/config.yaml @@ -1,25 +1,25 @@ queues: queue_1: - trs_id: '' - version_id: develop + trs_id: null + version_id: null wes_default: local wes_opts: - local workflow_attachments: - file://tests/testdata/md5sum.input - file://tests/testdata/dockstore-tool-md5sum.cwl - workflow_id: '' + workflow_id: null workflow_type: CWL workflow_url: file://tests/testdata/md5sum.cwl toolregistries: dockstore: - auth: '' - auth_type: '' + auth: null + auth_type: null host: dockstore.org:8443 proto: https workflowservices: local: - auth: '' - auth_type: '' + auth: null + auth_type: null host: 0.0.0.0:8080 proto: http From 3cd718ec34c8dcf7f604a8788960ea81cd1bcd81 Mon Sep 17 00:00:00 2001 From: James Eddy Date: Fri, 31 Aug 2018 09:29:45 -0700 Subject: [PATCH 3/5] actually remove config from version control --- synorchestrator/config.yaml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 synorchestrator/config.yaml diff --git a/synorchestrator/config.yaml b/synorchestrator/config.yaml deleted file mode 100644 index 3dd3edd..0000000 --- a/synorchestrator/config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -queues: - queue_1: - trs_id: null - version_id: null - wes_default: local - wes_opts: - - local - workflow_attachments: - - file://tests/testdata/md5sum.input - - file://tests/testdata/dockstore-tool-md5sum.cwl - workflow_id: null - workflow_type: CWL - workflow_url: file://tests/testdata/md5sum.cwl -toolregistries: - dockstore: - auth: null - auth_type: null - host: dockstore.org:8443 - proto: https -workflowservices: - local: - auth: null - auth_type: null - host: 0.0.0.0:8080 - proto: http From e4f08c9e7923e0482982290f39c9f8c58bc86bab Mon Sep 17 00:00:00 2001 From: James Eddy Date: Fri, 31 Aug 2018 10:05:36 -0700 Subject: [PATCH 4/5] fix: clean up formatting in config module, add WES hookup fxn Update 'show' function to use consistent naming and provide additional details about app config. Also provide function to connect a WES endpoint to one or more workflow queues. --- synorchestrator/config.py | 67 ++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/synorchestrator/config.py b/synorchestrator/config.py index 2abcabe..631ecb5 100644 --- a/synorchestrator/config.py +++ b/synorchestrator/config.py @@ -1,5 +1,5 @@ """ -The orchestrator config file has three sections: eval, trs, and wes. +The orchestrator config file has three sections: queues, trs, and wes. This provides functions to save and get values into these three sections in the file. """ @@ -37,13 +37,12 @@ def add_queue(queue_id, wes_default='local', wes_opts=None): """ - Register a Synapse evaluation queue to the orchestrator's + Register a workflow evaluation queue to the orchestrator's scope of work. - - :param eval_id: integer ID of a Synapse evaluation queue """ - # TODO: require either workflow url/attachments OR - # TRS information for retrieval + if not wf_id and not wf_url: + raise ValueError("One of either 'wf_id' or 'wf_url' must be specified.") + wes_opts = [wes_default] if wes_opts is None else wes_opts config = {'workflow_type': wf_type, 'trs_id': trs_id, @@ -61,7 +60,7 @@ def add_toolregistry(service, auth, auth_type, host, proto): Register a Tool Registry Service endpoint to the orchestrator's search space for workflows. - :param trs_id: string ID of TRS endpoint (e.g., 'Dockstore') + :param trs_id: string ID of TRS endpoint (e.g., 'dockstore') """ config = {'auth': auth, 'auth_type': auth, @@ -70,12 +69,12 @@ def add_toolregistry(service, auth, auth_type, host, proto): set_yaml('toolregistries', service, config) -def add_workflowservice(service, auth, auth_type, host, proto): +def add_workflowservice(service, host, auth=None, auth_type=None, proto='https'): """ Register a Workflow Execution Service endpoint to the orchestrator's available environment options. - :param wes_id: string ID of WES endpoint (e.g., 'workflow-service') + :param wes_id: string ID of WES endpoint (e.g., 'local') """ config = {'auth': auth, 'auth_type': auth_type, @@ -84,6 +83,21 @@ def add_workflowservice(service, auth, auth_type, host, proto): set_yaml('workflowservices', service, config) +def add_wes_opt(queue_ids, wes_id, make_default=False): + """ + Add a WES endpoint to the execution options of the specified + workflow queues. + """ + if not isinstance(queue_ids, list): + queue_ids = [queue_ids] + for queue_id in queue_ids: + wf_config = queue_config()[queue_id] + wf_config['wes_opts'].append(wes_id) + if make_default: + wf_config['wes_default'] = wes_id + set_yaml('queues', queue_id, wf_config) + + def set_yaml(section, service, var2add): orchestrator_config = get_yaml(config_path) orchestrator_config.setdefault(section, {})[service] = var2add @@ -95,16 +109,39 @@ def show(): Show current application configuration. """ orchestrator_config = get_yaml(config_path) - workflows = '\n'.join('{}:\t{}\t[{}]'.format(k, orchestrator_config['queues'][k]['workflow_id'], orchestrator_config['queues'][k]['workflow_type']) for k in orchestrator_config['queues']) - trs = '\n'.join('{}: {}'.format(k, orchestrator_config['toolregistries'][k]['host']) for k in orchestrator_config['toolregistries']) - wes = '\n'.join('{}: {}'.format(k, orchestrator_config['workflowservices'][k]['host']) for k in orchestrator_config['workflowservices']) + queues = '\n'.join( + ('{}: {} ({})\n' + ' > workflow URL: {}\n' + ' > workflow type: {}\n' + ' > from TRS: {}\n' + ' > WES options: {}').format( + k, + orchestrator_config['queues'][k]['workflow_id'], + orchestrator_config['queues'][k]['version_id'], + orchestrator_config['queues'][k]['workflow_url'], + orchestrator_config['queues'][k]['workflow_type'], + orchestrator_config['queues'][k]['trs_id'], + orchestrator_config['queues'][k]['wes_opts'] + ) + for k in orchestrator_config['queues'] + ) + trs = '\n'.join('{}: {}'.format( + k, + orchestrator_config['toolregistries'][k]['host']) + for k in orchestrator_config['toolregistries'] + ) + wes = '\n'.join('{}: {}'.format( + k, + orchestrator_config['workflowservices'][k]['host']) + for k in orchestrator_config['workflowservices'] + ) display = heredoc(''' Orchestrator options: Workflow Evaluation Queues - (queue ID: workflow ID [workflow type]) + (queue ID: workflow ID [version]) --------------------------------------------------------------------------- - {evals} + {queues} Tool Registries (TRS ID: host address) @@ -115,7 +152,7 @@ def show(): (WES ID: host address) --------------------------------------------------------------------------- {wes} - ''', {'evals': evals, + ''', {'queues': queues, 'trs': trs, 'wes': wes}) print(display) From 3070eb4b580e99212dd3bdccae128be55d0df098 Mon Sep 17 00:00:00 2001 From: James Eddy Date: Fri, 31 Aug 2018 11:13:41 -0700 Subject: [PATCH 5/5] chore: remove default config and submission queue files Autogenerate default settings for each file when respective modules are loaded. --- .gitignore | 1 + synorchestrator/config.py | 52 +++++++++++++++++++++++---- synorchestrator/queue.py | 3 +- synorchestrator/submission_queue.json | 9 ----- 4 files changed, 48 insertions(+), 17 deletions(-) delete mode 100644 synorchestrator/submission_queue.json diff --git a/.gitignore b/.gitignore index 31cd58e..f02a2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -113,5 +113,6 @@ ENV/ *.config *.evals *config.yaml +*submission_queue.json .DS_Store diff --git a/synorchestrator/config.py b/synorchestrator/config.py index 631ecb5..3b8d7c3 100644 --- a/synorchestrator/config.py +++ b/synorchestrator/config.py @@ -12,7 +12,41 @@ logger = logging.getLogger(__name__) +def _default_config(): + """ + Create default app config, if not existing. + """ + config = { + 'queues': { + 'queue_1': { + 'trs_id': None, + 'version_id': None, + 'wes_default': 'local', + 'wes_opts': ['local'], + 'workflow_attachments': [ + 'file://tests/testdata/md5sum.input', + 'file://tests/testdata/dockstore-tool-md5sum.cwl'], + 'workflow_id': None, + 'workflow_type': 'CWL', + 'workflow_url': 'file://tests/testdata/md5sum.cwl'}}, + 'toolregistries': { + 'dockstore': { + 'auth': None, + 'auth_type': None, + 'host': 'dockstore.org:8443', + 'proto': 'https'}}, + 'workflowservices': { + 'local': { + 'auth': None, + 'auth_type': None, + 'host': '0.0.0.0:8080', + 'proto': 'http'}}} + save_yaml(config_path, config) + + config_path = os.path.join(os.path.dirname(__file__), 'config.yaml') +if not os.path.exists(config_path): + _default_config() def queue_config(): @@ -41,7 +75,8 @@ def add_queue(queue_id, scope of work. """ if not wf_id and not wf_url: - raise ValueError("One of either 'wf_id' or 'wf_url' must be specified.") + raise ValueError( + "One of either 'wf_id' or 'wf_url' must be specified.") wes_opts = [wes_default] if wes_opts is None else wes_opts config = {'workflow_type': wf_type, @@ -115,24 +150,24 @@ def show(): ' > workflow type: {}\n' ' > from TRS: {}\n' ' > WES options: {}').format( - k, + k, orchestrator_config['queues'][k]['workflow_id'], orchestrator_config['queues'][k]['version_id'], orchestrator_config['queues'][k]['workflow_url'], orchestrator_config['queues'][k]['workflow_type'], orchestrator_config['queues'][k]['trs_id'], orchestrator_config['queues'][k]['wes_opts'] - ) + ) for k in orchestrator_config['queues'] ) trs = '\n'.join('{}: {}'.format( - k, - orchestrator_config['toolregistries'][k]['host']) + k, + orchestrator_config['toolregistries'][k]['host']) for k in orchestrator_config['toolregistries'] ) wes = '\n'.join('{}: {}'.format( - k, - orchestrator_config['workflowservices'][k]['host']) + k, + orchestrator_config['workflowservices'][k]['host']) for k in orchestrator_config['workflowservices'] ) display = heredoc(''' @@ -156,3 +191,6 @@ def show(): 'trs': trs, 'wes': wes}) print(display) + + + diff --git a/synorchestrator/queue.py b/synorchestrator/queue.py index 5f0cecc..d63c4d3 100644 --- a/synorchestrator/queue.py +++ b/synorchestrator/queue.py @@ -8,7 +8,8 @@ submission_queue = os.path.join(os.path.dirname(__file__), 'submission_queue.json') - +if not os.path.exists(submission_queue): + save_json(submission_queue, {}) def create_queue(): pass diff --git a/synorchestrator/submission_queue.json b/synorchestrator/submission_queue.json deleted file mode 100644 index ff98656..0000000 --- a/synorchestrator/submission_queue.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "mock_queue_1_checker": { - "300830212830955405": { - "status": "RECEIVED", - "wes_id": "local", - "data": "" - } - } -} \ No newline at end of file