diff --git a/.gitignore b/.gitignore index 6d9997a..f02a2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -112,5 +112,7 @@ ENV/ # local config files *.config *.evals +*config.yaml +*submission_queue.json .DS_Store diff --git a/synorchestrator/config.py b/synorchestrator/config.py index 2abcabe..3b8d7c3 100644 --- a/synorchestrator/config.py +++ b/synorchestrator/config.py @@ -1,5 +1,5 @@ """ -The orchestrator config file has three sections: eval, trs, and wes. +The orchestrator config file has three sections: queues, trs, and wes. This provides functions to save and get values into these three sections in the file. """ @@ -12,7 +12,41 @@ logger = logging.getLogger(__name__) +def _default_config(): + """ + Create default app config, if not existing. + """ + config = { + 'queues': { + 'queue_1': { + 'trs_id': None, + 'version_id': None, + 'wes_default': 'local', + 'wes_opts': ['local'], + 'workflow_attachments': [ + 'file://tests/testdata/md5sum.input', + 'file://tests/testdata/dockstore-tool-md5sum.cwl'], + 'workflow_id': None, + 'workflow_type': 'CWL', + 'workflow_url': 'file://tests/testdata/md5sum.cwl'}}, + 'toolregistries': { + 'dockstore': { + 'auth': None, + 'auth_type': None, + 'host': 'dockstore.org:8443', + 'proto': 'https'}}, + 'workflowservices': { + 'local': { + 'auth': None, + 'auth_type': None, + 'host': '0.0.0.0:8080', + 'proto': 'http'}}} + save_yaml(config_path, config) + + config_path = os.path.join(os.path.dirname(__file__), 'config.yaml') +if not os.path.exists(config_path): + _default_config() def queue_config(): @@ -37,13 +71,13 @@ def add_queue(queue_id, wes_default='local', wes_opts=None): """ - Register a Synapse evaluation queue to the orchestrator's + Register a workflow evaluation queue to the orchestrator's scope of work. - - :param eval_id: integer ID of a Synapse evaluation queue """ - # TODO: require either workflow url/attachments OR - # TRS information for retrieval + if not wf_id and not wf_url: + raise ValueError( + "One of either 'wf_id' or 'wf_url' must be specified.") + wes_opts = [wes_default] if wes_opts is None else wes_opts config = {'workflow_type': wf_type, 'trs_id': trs_id, @@ -61,7 +95,7 @@ def add_toolregistry(service, auth, auth_type, host, proto): Register a Tool Registry Service endpoint to the orchestrator's search space for workflows. - :param trs_id: string ID of TRS endpoint (e.g., 'Dockstore') + :param trs_id: string ID of TRS endpoint (e.g., 'dockstore') """ config = {'auth': auth, 'auth_type': auth, @@ -70,12 +104,12 @@ def add_toolregistry(service, auth, auth_type, host, proto): set_yaml('toolregistries', service, config) -def add_workflowservice(service, auth, auth_type, host, proto): +def add_workflowservice(service, host, auth=None, auth_type=None, proto='https'): """ Register a Workflow Execution Service endpoint to the orchestrator's available environment options. - :param wes_id: string ID of WES endpoint (e.g., 'workflow-service') + :param wes_id: string ID of WES endpoint (e.g., 'local') """ config = {'auth': auth, 'auth_type': auth_type, @@ -84,6 +118,21 @@ def add_workflowservice(service, auth, auth_type, host, proto): set_yaml('workflowservices', service, config) +def add_wes_opt(queue_ids, wes_id, make_default=False): + """ + Add a WES endpoint to the execution options of the specified + workflow queues. + """ + if not isinstance(queue_ids, list): + queue_ids = [queue_ids] + for queue_id in queue_ids: + wf_config = queue_config()[queue_id] + wf_config['wes_opts'].append(wes_id) + if make_default: + wf_config['wes_default'] = wes_id + set_yaml('queues', queue_id, wf_config) + + def set_yaml(section, service, var2add): orchestrator_config = get_yaml(config_path) orchestrator_config.setdefault(section, {})[service] = var2add @@ -95,16 +144,39 @@ def show(): Show current application configuration. """ orchestrator_config = get_yaml(config_path) - workflows = '\n'.join('{}:\t{}\t[{}]'.format(k, orchestrator_config['queues'][k]['workflow_id'], orchestrator_config['queues'][k]['workflow_type']) for k in orchestrator_config['queues']) - trs = '\n'.join('{}: {}'.format(k, orchestrator_config['toolregistries'][k]['host']) for k in orchestrator_config['toolregistries']) - wes = '\n'.join('{}: {}'.format(k, orchestrator_config['workflowservices'][k]['host']) for k in orchestrator_config['workflowservices']) + queues = '\n'.join( + ('{}: {} ({})\n' + ' > workflow URL: {}\n' + ' > workflow type: {}\n' + ' > from TRS: {}\n' + ' > WES options: {}').format( + k, + orchestrator_config['queues'][k]['workflow_id'], + orchestrator_config['queues'][k]['version_id'], + orchestrator_config['queues'][k]['workflow_url'], + orchestrator_config['queues'][k]['workflow_type'], + orchestrator_config['queues'][k]['trs_id'], + orchestrator_config['queues'][k]['wes_opts'] + ) + for k in orchestrator_config['queues'] + ) + trs = '\n'.join('{}: {}'.format( + k, + orchestrator_config['toolregistries'][k]['host']) + for k in orchestrator_config['toolregistries'] + ) + wes = '\n'.join('{}: {}'.format( + k, + orchestrator_config['workflowservices'][k]['host']) + for k in orchestrator_config['workflowservices'] + ) display = heredoc(''' Orchestrator options: Workflow Evaluation Queues - (queue ID: workflow ID [workflow type]) + (queue ID: workflow ID [version]) --------------------------------------------------------------------------- - {evals} + {queues} Tool Registries (TRS ID: host address) @@ -115,7 +187,10 @@ def show(): (WES ID: host address) --------------------------------------------------------------------------- {wes} - ''', {'evals': evals, + ''', {'queues': queues, 'trs': trs, 'wes': wes}) print(display) + + + diff --git a/synorchestrator/config.yaml b/synorchestrator/config.yaml deleted file mode 100644 index 6d66b1f..0000000 --- a/synorchestrator/config.yaml +++ /dev/null @@ -1,49 +0,0 @@ -queues: - queue_1: - trs_id: '' - version_id: develop - wes_default: local - wes_opts: - - local - workflow_attachments: - - file://tests/testdata/md5sum.input - - file://tests/testdata/dockstore-tool-md5sum.cwl - workflow_id: '' - workflow_type: CWL - workflow_url: file://tests/testdata/md5sum.cwl - queue_2: - trs_id: dockstore - version_id: develop - wes_default: local - wes_opts: - - local - workflow_attachments: - - !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/md5sum/md5sum-tool.cwl' - workflow_id: github.com/dockstore-testing/md5sum-checker - workflow_type: CWL - workflow_url: !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/md5sum/md5sum-workflow.cwl' - queue_2_checker: - trs_id: dockstore - version_id: develop - wes_default: local - wes_opts: - - local - workflow_attachments: - - !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/checker/md5sum-checker.cwl' - - !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/md5sum/md5sum-tool.cwl' - - !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/md5sum/md5sum-workflow.cwl' - workflow_id: !!python/unicode 'github.com/dockstore-testing/md5sum-checker/_cwl_checker' - workflow_type: CWL - workflow_url: !!python/unicode 'https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/develop/checker-workflow-wrapping-workflow.cwl' -toolregistries: - dockstore: - auth: '' - auth_type: '' - host: dockstore.org:8443 - proto: https -workflowservices: - local: - auth: '' - auth_type: '' - host: 0.0.0.0:8080 - proto: http diff --git a/synorchestrator/queue.py b/synorchestrator/queue.py index 5f0cecc..d63c4d3 100644 --- a/synorchestrator/queue.py +++ b/synorchestrator/queue.py @@ -8,7 +8,8 @@ submission_queue = os.path.join(os.path.dirname(__file__), 'submission_queue.json') - +if not os.path.exists(submission_queue): + save_json(submission_queue, {}) def create_queue(): pass diff --git a/synorchestrator/submission_queue.json b/synorchestrator/submission_queue.json deleted file mode 100644 index ff98656..0000000 --- a/synorchestrator/submission_queue.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "mock_queue_1_checker": { - "300830212830955405": { - "status": "RECEIVED", - "wes_id": "local", - "data": "" - } - } -} \ No newline at end of file