refactor: submit checker jobs as params, not full WES requests

Update functions and logic in the testbed module to prepare and
create new workflow checker jobs as test params (JSON), submitted
to evaluation queues. Information about how to run the actual
workflow and post the WES request will be associated directly
with the evaluation queue.

See #33

synorchestrator/testbed.py

@@ -1,13 +1,77 @@
+#!/usr/bin/env python
+"""
+"""
+import logging
+import urllib
+import re
 
+from synorchestrator.config import eval_config as queue_config
+from synorchestrator.config import trs_config
+
+from synorchestrator.trs.wrapper import TRS
+from synorchestrator.eval import create_submission
+from synorchestrator.orchestrator import run_queue
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+
+def testbed_config():
+    pass
+
+
+def create_queue():
+    pass
+
+
+def get_checker_id(trs, workflow_id):
+    """
+    Return entry for the specified workflow's "checker workflow."
+    """
+    target_workflow = trs.get_workflow(id=workflow_id)
+    checker_url = urllib.unquote(target_workflow['checker_url'])
+    checker_id = re.sub('^.*#workflow/', '', checker_url)
+    logger.info("found checker workflow: {}".format(checker_id))
+    return checker_id
+
+
+def check_workflow(workflow_id, wes_id):
+    """
+    Run checker workflow in a single environment.
+    """
+    wf_config = testbed_config()[workflow_id]
+    logger.info("Preparing checker workflow run request for '{}' from  '{}''"
+                .format(workflow_id, wf_config['trs_id']))
+
+    trs_instance = TRS(**trs_config()[wf_config['trs_id']])
+    checker_id = get_checker_id(trs_instance, workflow_id)
+
+    queue_id = create_queue(workflow={'trs_id': wf_config['trs_id'],
+                                      'id': checker_id,  
+                                      'version_id': wf_config['version_id'],
+                                      'type': wf_config['type']})
+
+    checker_job = trs_instance.get_workflow_tests(id=checker_id,
+                                         version_id=wf_config['version_id'],
+                                         type=wf_config['type'])[0]
+
+    submission_id = create_submission(queue_id=queue_id, 
+                                      submission_data=checker_job, 
+                                      wes_id=wes_id, 
+                                      type='checker')
+    return run_queue(queue_id)
+
+
+def check_all(workflow_wes_map):
+    """
+    Check workflows for multiple workflows in multiple environments
+    (cross product of workflows, workflow service endpoints).
+    """
+    submission_logs = [check_workflow(workflow_id, wes_id)
+                       for workflow_id in workflow_wes_map
+                       for wes_id in workflow_wes_map[workflow_id]]
+    return submission_logs
 
-# def get_workflow_checker(self, id):
-#     """
-#     Return entry for the specified workflow's "checker workflow."
-#     """
-#     checker_url = urllib.unquote(self.get_workflow(id=id)['checker_url'])
-#     checker_id = re.sub('^.*#workflow/', '', checker_url)
-#     logger.info("found checker workflow: {}".format(checker_id))
-#     return self.get_workflow(id=checker_id)
 
 # def post_verification(self, id, version_id, type, relative_path, requests):
 #     """

tests/test_testbed.py

@@ -0,0 +1,149 @@
+import mock
+import pytest
+
+from synorchestrator.testbed import get_checker_id
+from synorchestrator.testbed import check_workflow
+from synorchestrator.testbed import check_all
+
+
+@pytest.fixture()
+def mock_testbed_config():
+    mock_testbed_config = {
+        'mock_wf': {
+            'trs_id': 'mock_trs',
+            'version_id': '',
+            'type': '',
+            'wes': []
+        }
+    }
+    yield mock_testbed_config
+
+
+@pytest.fixture()
+def mock_queue_config():
+    mock_queue_config = {
+        'mock_queue': {
+            'workflow_trs_id': 'mock_trs',
+            'workflow_id': 'mock_wf',
+            'workflow_version_id': '',
+            'workflow_type': ''
+        }
+    }
+    yield mock_queue_config
+
+
+@pytest.fixture()
+def mock_trs_config():
+    mock_trs_config = {
+        'mock_trs': {
+            'auth': 'auth_token',
+            'host': '0.0.0.0:8080',
+            'proto': 'https'
+        }
+    }
+    yield mock_trs_config
+
+
+@pytest.fixture()
+def mock_trs(request):
+    mock_trs = mock.Mock(name='mock TRS')
+    with mock.patch('synorchestrator.trs.wrapper.TRS', 
+                    autospec=True, spec_set=True):
+        yield mock_trs
+
+
+def test_get_checker_id(mock_trs_config, mock_trs,  monkeypatch):
+    monkeypatch.setattr('synorchestrator.testbed.TRS', 
+                        lambda host,auth,proto: mock_trs)
+
+    mock_checker_url = '/%23workflow%2Fmock_wf%2F_cwl_checker'
+    mock_trs.get_workflow.return_value = {'checker_url': mock_checker_url}
+    mock_checker_id = 'mock_wf/_cwl_checker'
+
+    test_checker_id = get_checker_id(mock_trs, 'mock_wf')
+
+    assert test_checker_id == mock_checker_id
+
+
+def test_check_queue(mock_testbed_config,
+                     mock_queue_config, 
+                     mock_trs_config, 
+                     mock_trs, 
+                     monkeypatch):
+    monkeypatch.setattr('synorchestrator.testbed.testbed_config', 
+                        lambda: mock_testbed_config)
+    monkeypatch.setattr('synorchestrator.testbed.queue_config', 
+                        lambda: mock_queue_config)
+    monkeypatch.setattr('synorchestrator.testbed.trs_config', 
+                        lambda: mock_trs_config)
+    monkeypatch.setattr('synorchestrator.testbed.TRS', 
+                        lambda host,auth,proto: mock_trs)                        
+    monkeypatch.setattr('synorchestrator.testbed.get_checker_id', 
+                        lambda x,y: 'mock_wf_checker')
+    monkeypatch.setattr('synorchestrator.testbed.create_queue', 
+                        lambda workflow: 'mock_queue')
+    monkeypatch.setattr('synorchestrator.testbed.create_submission', 
+                        lambda queue_id,submission_data,wes_id,type: 'mock_sub')
+
+    mock_trs.get_workflow_tests.return_value = [{'content': '', 'url': ''}]
+
+    mock_submission_log = {
+        'mock_wf': {
+            'mock_sub': {
+                'queue_id': 'mock_queue',
+                'job': '',
+                'wes_id': '',
+                'run_id': 'mock_run',
+                'status': 'QUEUED',
+                'start_time': ''
+            }
+        }
+    }
+    monkeypatch.setattr('synorchestrator.testbed.run_queue', 
+                        lambda x: mock_submission_log)
+
+    test_submission_log = check_workflow(workflow_id='mock_wf', 
+                                         wes_id='mock_wes')
+
+    assert test_submission_log == mock_submission_log
+
+
+def test_check_all(mock_testbed_config, monkeypatch):
+    monkeypatch.setattr('synorchestrator.testbed.queue_config', 
+                        lambda: mock_queue_config)
+
+    mock_submission_logs = {
+        'mock_wes_1': {
+            'mock_wf': {
+                'mock_sub': {
+                    'queue_id': 'mock_queue',
+                    'job': '',
+                    'wes_id': 'mock_wes_1',
+                    'run_id': 'mock_run',
+                    'status': 'QUEUED',
+                    'start_time': ''
+                }
+            }
+        },
+        'mock_wes_2': {
+            'mock_wf': {
+                'mock_sub': {
+                    'queue_id': 'mock_queue',
+                    'job': '',
+                    'wes_id': 'mock_wes_2',
+                    'run_id': 'mock_run',
+                    'status': 'QUEUED',
+                    'start_time': ''
+                }
+            }
+        }
+    }
+    monkeypatch.setattr('synorchestrator.testbed.check_workflow', 
+                        lambda x,y: mock_submission_logs[y])
+
+    mock_workflow_wes_map = {
+        'mock_wf': ['mock_wes_1', 'mock_wes_2']
+    }
+    test_submission_logs = check_all(mock_workflow_wes_map)
+    assert all([log in mock_submission_logs.values() 
+                for log in test_submission_logs])