diff --git a/.travis.yml b/.travis.yml index 586bb41..589e47a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: - pip install sphinx sphinx-bootstrap-theme coveralls ipython[all]==2.4.1 # Installing the simplify REST api branch - change once it has been merged to master # - pip install https://github.com/biocore/qiita/archive/master.zip - - pip install https://github.com/biocore/qiita/archive/simplify-restAPI.zip + - pip install https://github.com/biocore/qiita/archive/plugin-installation.zip - ipython profile create qiita-general --parallel - qiita-env start_cluster qiita-general - qiita-env make --no-load-ontologies diff --git a/qiita_client/__init__.py b/qiita_client/__init__.py index 3ab39d8..a582196 100644 --- a/qiita_client/__init__.py +++ b/qiita_client/__init__.py @@ -9,6 +9,9 @@ from .exceptions import (QiitaClientError, NotFoundError, BadRequestError, ForbiddenError) from .qiita_client import QiitaClient, ArtifactInfo +from .plugin import (QiitaCommand, QiitaPlugin, QiitaTypePlugin, + QiitaArtifactType) __all__ = ["QiitaClient", "QiitaClientError", "NotFoundError", - "BadRequestError", "ForbiddenError", "ArtifactInfo"] + "BadRequestError", "ForbiddenError", "ArtifactInfo", "QiitaCommand", + "QiitaPlugin", "QiitaTypePlugin", "QiitaArtifactType"] diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py new file mode 100644 index 0000000..03c23e4 --- /dev/null +++ b/qiita_client/plugin.py @@ -0,0 +1,352 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +import traceback +import sys +from string import ascii_letters, digits +from random import SystemRandom +from os.path import exists, join, expanduser +from os import makedirs, environ +from future import standard_library +from json import dumps +import urllib + +from qiita_client import QiitaClient + +with standard_library.hooks(): + from configparser import ConfigParser + + +class QiitaCommand(object): + """A plugin command + + Parameters + ---------- + name : str + The command name + description : str + The command description + function : callable + The function that executes the command. Should be a callable that + conforms to the signature: + `(bool, str, [ArtifactInfo] = function(qclient, job_id, job_parameters, + output_dir)` + where qclient is an instance of QiitaClient, job_id is a string with + the job identifier, job_parameters is a dictionary with the parameters + of the command and output_dir is a string with the output directory. + The function should return a boolean indicating if the command was + executed successfully or not, a string containing a message in case + of error, and a list of ArtifactInfo objects in case of success. + required_parameters : dict of {str: (str, list of str)} + The required parameters of the command, keyed by parameter name. The + values should be a 2-tuple in which the first element is the parameter + type, and the second parameter is the list of subtypes (if applicable) + optional_parameters : dict of {str: (str, str)} + The optional parameters of the command, keyed by parameter name. The + values should be a 2-tuple in which the first element is the parameter + name, and the second paramater is the default value + outputs : dict of {str: str} + The description of the outputs that this command generated. The + format is: {output_name: artifact_type} + default_parameter_sets : dict of {str: dict of {str: str}} + The default parameter sets of the command, keyed by parameter set name. + The values should be a dictionary in which keys are the parameter names + and values are the specific value for each parameter + + Raises + ------ + TypeError + If `function` is not callable + ValueError + If `function` does not accept 4 parameters + """ + def __init__(self, name, description, function, required_parameters, + optional_parameters, outputs, default_parameter_sets=None): + self.name = name + self.description = description + + # Make sure that `function` is callable + if not callable(function): + raise TypeError( + "Couldn't create command '%s': the provided function is not " + "callable (type: %s)" % (name, type(function))) + + # `function` will be called with the following Parameters + # qclient, job_id, job_parameters, output_dir + # Make sure that `function` can receive 4 parameters + if function.__code__.co_argcount != 4: + raise ValueError( + "Couldn't register command '%s': the provided function does " + "not accept 4 parameters (number of parameters: %d)" + % (name, function.__code__.co_argcount)) + + self.function = function + self.required_parameters = required_parameters + self.optional_parameters = optional_parameters + self.default_parameter_sets = default_parameter_sets + self.outputs = outputs + + def __call__(self, qclient, server_url, job_id, output_dir): + return self.function(qclient, server_url, job_id, output_dir) + + +class QiitaArtifactType(object): + """A Qiita artifact type + + Parameters + ---------- + name : str + The artifact type name + description : str + The artifact type description + can_be_submitted_to_ebi : bool + Whether the artifact type can be submitted to EBI or not + can_be_submitted_to_vamps : bool + Whether the artifact type can be submitted to VAMPS or not + filepath_types : list of (str, bool) + The list filepath types that the new artifact type supports, and + if they're required or not in an artifact instance of this type""" + def __init__(self, name, description, can_be_submitted_to_ebi, + can_be_submitted_to_vamps, filepath_types): + self.name = name + self.description = description + self.ebi = can_be_submitted_to_ebi + self.vamps = can_be_submitted_to_vamps + self.fp_types = filepath_types + + +class BaseQiitaPlugin(object): + def __init__(self, name, version, description, publications=None): + self.name = name + self.version = version + self.description = description + self.publications = dumps(publications) if publications else "" + + # Will hold the different commands + self.task_dict = {} + + # The configuration file + conf_dir = environ.get( + 'QIITA_PLUGINS_DIR', join(expanduser('~'), '.qiita_plugins')) + self.conf_fp = join(conf_dir, "%s_%s.conf" % (self.name, self.version)) + + def generate_config(self, env_script, start_script, server_cert=None): + """Generates the plugin configuration file + + Parameters + ---------- + env_script : str + The CLI call used to load the environment in which the plugin is + installed + start_script : str + The script used to start the plugin + server_cert : str, optional + If the Qiita server used does not have a valid certificate, the + path to the Qiita certificate so the plugin can connect over + HTTPS to it + """ + sr = SystemRandom() + chars = ascii_letters + digits + client_id = ''.join(sr.choice(chars) for i in range(50)) + client_secret = ''.join(sr.choice(chars) for i in range(255)) + + server_cert = server_cert if server_cert else "" + + with open(self.conf_fp, 'w') as f: + f.write(CONF_TEMPLATE % (self.name, self.version, self.description, + env_script, start_script, + self._plugin_type, self.publications, + server_cert, client_id, client_secret)) + + def _register_command(self, command): + """Registers a command in the plugin + + Parameters + ---------- + command: QiitaCommand + The command to be added to the plugin + """ + self.task_dict[command.name] = command + + def _register(self, qclient): + """Registers the plugin information in Qiita""" + # Get the command information from qiita + info = qclient.get('/qiita_db/plugins/%s/%s/' + % (self.name, self.version)) + + for cmd in self.task_dict.values(): + if cmd.name in info['commands']: + qclient.post('/qiita_db/plugins/%s/%s/commands/%s/activate/' + % (self.name, self.version, + urllib.parse.quote(cmd.name))) + else: + req_params = { + k: v if v[0] != 'artifact' else ['artifact:%s' + % dumps(v[1]), None] + for k, v in cmd.required_parameters.items()} + + data = {'name': cmd.name, + 'description': cmd.description, + 'required_parameters': dumps(req_params), + 'optional_parameters': dumps(cmd.optional_parameters), + 'default_parameter_sets': dumps( + cmd.default_parameter_sets)} + qclient.post('/qiita_db/plugins/%s/%s/commands/' + % (self.name, self.version), data=data) + + def __call__(self, server_url, job_id, output_dir): + """Runs the plugin and executed the assigned task + + Parameters + ---------- + server_url : str + The url of the server + job_id : str + The job id + output_dir : str + The output directory + + Raises + ------ + RuntimeError + If there is a problem gathering the job information + """ + # Set up the Qiita Client + config = ConfigParser() + with open(self.conf_fp, 'U') as conf_file: + config.readfp(conf_file) + + qclient = QiitaClient(server_url, config.get('oauth2', 'CLIENT_ID'), + config.get('oauth2', 'CLIENT_SECRET'), + server_cert=config.get('oauth2', 'SERVER_CERT')) + + if job_id == 'register': + self._register(qclient) + else: + # Request job information. If there is a problem retrieving the job + # information, the QiitaClient already raises an error + job_info = qclient.get_job_info(job_id) + # Starting the heartbeat + qclient.start_heartbeat(job_id) + # Execute the given task + task_name = job_info['command'] + task = self.task_dict[task_name] + + if not exists(output_dir): + makedirs(output_dir) + try: + success, artifacts_info, error_msg = task( + qclient, job_id, job_info['parameters'], output_dir) + except Exception: + exc_str = repr(traceback.format_exception(*sys.exc_info())) + error_msg = ("Error executing %s:\n%s" % (task_name, exc_str)) + success = False + artifacts_info = None + # The job completed + qclient.complete_job(job_id, success, error_msg=error_msg, + artifacts_info=artifacts_info) + + +class QiitaTypePlugin(BaseQiitaPlugin): + """Represents a Qiita Type Plugin + + Parameters + ---------- + name : str + The plugin name + version : str + The plugin version + description : str + The plugin description + validate_func : callable + The function used to validate artifacts + html_generator_func : callable + The function used to generate the HTML generator + artifact_types : list of QiitaArtifactType + The artifact types defined in this plugin + + Notes + ----- + Both `validate_func` and `html_generator_func` should be a callable + that conforms to the signature: + `(bool, str, [ArtifactInfo] = function(qclient, job_id, job_parameters, + output_dir)` + where qclient is an instance of QiitaClient, job_id is a string with + the job identifier, job_parameters is a dictionary with the parameters + of the command and output_dir is a string with the output directory. + The function should return a boolean indicating if the command was + executed successfully or not, a string containing a message in case + of error, and a list of ArtifactInfo objects in case of success. + """ + _plugin_type = "artifact definition" + + def __init__(self, name, version, description, validate_func, + html_generator_func, artifact_types, publications=None): + super(QiitaTypePlugin, self).__init__(name, version, description, + publications=publications) + + self.artifact_types = artifact_types + + val_cmd = QiitaCommand( + 'Validate', 'Validates a new artifact', validate_func, + {'template': ('prep_template', None), + 'files': ('string', None), + 'artifact_type': ('string', None)}, {}, None) + + self._register_command(val_cmd) + + html_cmd = QiitaCommand( + 'Generate HTML summary', 'Generates the HTML summary', + html_generator_func, + {'input_data': ('artifact', + [a.name for a in self.artifact_types])}, {}, None) + + self._register_command(html_cmd) + + def _register(self, qclient): + """Registers the plugin information in Qiita""" + for at in self.artifact_types: + data = {'type_name': at.name, + 'description': at.description, + 'can_be_submitted_to_ebi': at.ebi, + 'can_be_submitted_to_vamps': at.vamps, + 'filepath_types': dumps(at.fp_types)} + qclient.post('/qiita_db/artifacts/types/', data=data) + + super(QiitaTypePlugin, self)._register(qclient) + + +class QiitaPlugin(BaseQiitaPlugin): + """Represents a Qiita Plugin""" + + _plugin_type = "artifact transformation" + + def register_command(self, command): + """Registers a command in the plugin + + Parameters + ---------- + command: QiitaCommand + The command to be added to the plugin + """ + self._register_command(command) + + +CONF_TEMPLATE = """[main] +NAME = %s +VERSION = %s +DESCRIPTION = %s +ENVIRONMENT_SCRIPT = %s +START_SCRIPT = %s +PLUGIN_TYPE = %s +PUBLICATIONS = %s + +[oauth2] +SERVER_CERT = %s +CLIENT_ID = %s +CLIENT_SECRET = %s""" diff --git a/qiita_client/testing.py b/qiita_client/testing.py new file mode 100644 index 0000000..7acfd83 --- /dev/null +++ b/qiita_client/testing.py @@ -0,0 +1,29 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from unittest import TestCase +from os import environ + +from qiita_client import QiitaClient + + +class PluginTestCase(TestCase): + @classmethod + def setUpClass(cls): + cls.client_id = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4' + cls.client_secret = ('J7FfQ7CQdOxuKhQAf1eoGgBAE81Ns8Gu3EKaWFm3IO2JKh' + 'AmmCWZuabe0O5Mp28s1') + cls.server_cert = environ.get('QIITA_SERVER_CERT', None) + cls.qclient = QiitaClient("https://localhost:21174", cls.client_id, + cls.client_secret, + server_cert=cls.server_cert) + + @classmethod + def tearDownClass(cls): + # Reset the test database + cls.qclient.post("/apitest/reset/") diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py new file mode 100644 index 0000000..18d0212 --- /dev/null +++ b/qiita_client/tests/test_plugin.py @@ -0,0 +1,217 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from unittest import TestCase, main +from os.path import isdir, exists, basename, join +from os import remove +from shutil import rmtree +from json import dumps +from tempfile import mkdtemp + +from qiita_client.testing import PluginTestCase +from qiita_client import (QiitaPlugin, QiitaTypePlugin, QiitaCommand, + QiitaArtifactType, ArtifactInfo) + + +class QiitaCommandTest(TestCase): + def setUp(self): + self.exp_req = {'p1': ('artifact', ['FASTQ'])} + self.exp_opt = {'p2': ('boolean', 'False'), + 'p3': ('string', 'somestring')} + self.exp_dflt = {'dflt1': {'p2': 'True', 'p3': 'anotherstring'}} + self.exp_out = {'out1': 'BIOM'} + + def test_init(self): + # Create a test function + def func(a, b, c, d): + return 42 + + obs = QiitaCommand("Test cmd", "Some description", func, self.exp_req, + self.exp_opt, self.exp_out, self.exp_dflt) + self.assertEqual(obs.name, "Test cmd") + self.assertEqual(obs.description, "Some description") + self.assertEqual(obs.function, func) + self.assertEqual(obs.required_parameters, self.exp_req) + self.assertEqual(obs.optional_parameters, self.exp_opt) + self.assertEqual(obs.outputs, self.exp_out) + self.assertEqual(obs.default_parameter_sets, self.exp_dflt) + + with self.assertRaises(TypeError): + QiitaCommand("Name", "Desc", "func", self.exp_req, self.exp_opt, + self.exp_out, self.exp_dflt) + + def func(a, b, c): + return 42 + + with self.assertRaises(ValueError): + QiitaCommand("Name", "Desc", func, self.exp_req, self.exp_opt, + self.exp_out, self.exp_dflt) + + def test_call(self): + def func(a, b, c, d): + return 42 + + obs = QiitaCommand("Test cmd", "Some description", func, self.exp_req, + self.exp_opt, self.exp_out, self.exp_dflt) + self.assertEqual(obs('a', 'b', 'c', 'd'), 42) + + +class QiitaArtifactTypeTest(TestCase): + def test_init(self): + obs = QiitaArtifactType('Name', 'Description', False, True, + [('plain_text', False)]) + self.assertEqual(obs.name, 'Name') + self.assertEqual(obs.description, 'Description') + self.assertFalse(obs.ebi) + self.assertTrue(obs.vamps) + self.assertEqual(obs.fp_types, [('plain_text', False)]) + + +class QiitaTypePluginTest(PluginTestCase): + def setUp(self): + self.clean_up_fp = [] + + def tearDown(self): + for fp in self.clean_up_fp: + if exists(fp): + if isdir(fp): + rmtree(fp) + else: + remove(fp) + + def test_init(self): + def validate_func(a, b, c, d): + return 42 + + def html_generator_func(a, b, c, d): + return 42 + + atypes = [QiitaArtifactType('Name', 'Description', False, True, + [('plain_text', False)])] + obs = QiitaTypePlugin("NewPlugin", "1.0.0", "Description", + validate_func, html_generator_func, + atypes) + self.assertEqual(obs.name, "NewPlugin") + self.assertEqual(obs.version, "1.0.0") + self.assertEqual(obs.description, "Description") + self.assertEqual(set(obs.task_dict.keys()), + {'Validate', 'Generate HTML summary'}) + self.assertEqual(obs.task_dict['Validate'].function, validate_func) + self.assertEqual(obs.task_dict['Generate HTML summary'].function, + html_generator_func) + self.assertEqual(obs.artifact_types, atypes) + self.assertEqual(basename(obs.conf_fp), 'NewPlugin_1.0.0.conf') + + def test_generate_config(self): + def validate_func(a, b, c, d): + return 42 + + def html_generator_func(a, b, c, d): + return 42 + atypes = [QiitaArtifactType('Name', 'Description', False, True, + [('plain_text', False)])] + tester = QiitaTypePlugin("NewPlugin", "1.0.0", "Description", + validate_func, html_generator_func, atypes) + + tester.generate_config('env_script', 'start_script') + self.assertTrue(exists(tester.conf_fp)) + with open(tester.conf_fp, 'U') as f: + conf = f.readlines() + + exp_lines = ['[main]\n', + 'NAME = NewPlugin\n', + 'VERSION = 1.0.0\n', + 'DESCRIPTION = Description\n', + 'ENVIRONMENT_SCRIPT = env_script\n', + 'START_SCRIPT = start_script\n', + 'PLUGIN_TYPE = artifact definition\n', + 'PUBLICATIONS = \n', + '\n', + '[oauth2]\n', + 'SERVER_CERT = \n'] + # We will test the last 2 lines independently since they're variable + # in each test run + self.assertEqual(conf[:-2], exp_lines) + self.assertTrue(conf[-2].startswith('CLIENT_ID = ')) + self.assertTrue(conf[-1].startswith('CLIENT_SECRET = ')) + + def test_call(self): + def validate_func(a, b, c, d): + return 42 + + def html_generator_func(a, b, c, d): + return 42 + + # Test the install procedure + atypes = [QiitaArtifactType('Name', 'Description', False, True, + [('plain_text', False)])] + tester = QiitaTypePlugin("NewPlugin", "1.0.0", "Description", + validate_func, html_generator_func, atypes) + + # Generate the config file for the new plugin + tester.generate_config('env_script', 'start_script', + server_cert=self.server_cert) + # Ask Qiita to reload the plugins + self.qclient.post('/apitest/reload_plugins/') + + # Install the current plugin + tester("https://localhost:21174", 'register', 'ignored') + + # Check that it has been installed + obs = self.qclient.get('/qiita_db/plugins/NewPlugin/1.0.0/') + self.assertEqual(obs['name'], 'NewPlugin') + self.assertEqual(obs['version'], '1.0.0') + + +class QiitaPluginTest(PluginTestCase): + # Most of the functionility is being tested in the previous + # class. Here we are going to test that we can actually execute a job + def setUp(self): + self.outdir = mkdtemp() + + def tearDown(self): + rmtree(self.outdir) + + def test_call(self): + def func(qclient, job_id, job_params, working_dir): + fp = join(working_dir, 'test.fastq') + with open(fp, 'w') as f: + f.write('') + res = ArtifactInfo('out1', 'Demultiplexed', + [[fp, 'preprocessed_fastq']]) + return True, "", [res] + + tester = QiitaPlugin("NewPlugin", "0.0.1", "description") + cmd = QiitaCommand("NewCmd", "Desc", func, + {'p1': ('artifact', ['FASTQ'])}, + {'p2': ('string', 'dflt')}, + {'out1': 'Demultiplexed'}) + tester.register_command(cmd) + + tester.generate_config('env_script', 'start_script', + server_cert=self.server_cert) + self.qclient.post('/apitest/reload_plugins/') + tester("https://localhost:21174", 'register', 'ignored') + + obs = self.qclient.get('/qiita_db/plugins/NewPlugin/0.0.1/') + self.assertEqual(obs['name'], 'NewPlugin') + self.assertEqual(obs['version'], '0.0.1') + self.assertEqual(obs['commands'], ['NewCmd']) + + # Create a new job + data = {'command': dumps(['NewPlugin', '0.0.1', 'NewCmd']), + 'parameters': dumps({'p1': '1', 'p2': 'a'}), + 'status': 'queued'} + job_id = self.qclient.post('/apitest/processing_job/', + data=data)['job'] + tester("https://localhost:21174", job_id, self.outdir) + obs = self.qclient.get_job_info(job_id) + self.assertEqual(obs['status'], 'success') + +if __name__ == '__main__': + main() diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 5d19b6b..61d052b 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -14,6 +14,7 @@ from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) +from qiita_client.testing import PluginTestCase from qiita_client.exceptions import BadRequestError CLIENT_ID = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4' @@ -66,15 +67,7 @@ def test_format_payload_error(self): self.assertEqual(obs, exp) -class QiitaClientTests(TestCase): - @classmethod - def tearDownClass(cls): - # Reset the test database - server_cert = environ.get('QIITA_SERVER_CERT', None) - qclient = QiitaClient("https://localhost:21174", CLIENT_ID, - CLIENT_SECRET, server_cert=server_cert) - qclient.post("/apitest/reset/") - +class QiitaClientTests(PluginTestCase): def setUp(self): self.server_cert = environ.get('QIITA_SERVER_CERT', None) self.tester = QiitaClient("https://localhost:21174", CLIENT_ID, @@ -194,7 +187,7 @@ def test_complete_job(self): # Create a new job data = { 'user': 'demo@microbio.me', - 'command': 3, + 'command': dumps(['QIIME', '1.9.1', 'Pick closed-reference OTUs']), 'status': 'running', 'parameters': dumps({"reference": 1, "sortmerna_e_value": 1, diff --git a/setup.py b/setup.py index 2d26023..e85a992 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,6 @@ test_suite='nose.collector', packages=['qiita_client'], extras_require={'test': ["nose >= 0.10.1", "pep8"]}, - install_requires=['click >= 3.3', 'requests'], + install_requires=['click >= 3.3', 'requests', 'future'], classifiers=classifiers )