From c526dfc78e4fd6d5bd0737267f07c3e08c255323 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Mon, 26 Sep 2016 13:00:09 -0700 Subject: [PATCH 1/9] Adding plugin class --- qiita_client/__init__.py | 4 +- qiita_client/plugin.py | 171 ++++++++++++++++++++++++++++++ qiita_client/tests/test_plugin.py | 22 ++++ 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 qiita_client/plugin.py create mode 100644 qiita_client/tests/test_plugin.py diff --git a/qiita_client/__init__.py b/qiita_client/__init__.py index 3ab39d8..31c0752 100644 --- a/qiita_client/__init__.py +++ b/qiita_client/__init__.py @@ -9,6 +9,8 @@ from .exceptions import (QiitaClientError, NotFoundError, BadRequestError, ForbiddenError) from .qiita_client import QiitaClient, ArtifactInfo +from .plugin import QiitaPlugin, QiitaTypePlugin __all__ = ["QiitaClient", "QiitaClientError", "NotFoundError", - "BadRequestError", "ForbiddenError", "ArtifactInfo"] + "BadRequestError", "ForbiddenError", "ArtifactInfo", "QiitaPlugin", + "QiitaTypePlugin"] diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py new file mode 100644 index 0000000..ada341c --- /dev/null +++ b/qiita_client/plugin.py @@ -0,0 +1,171 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +import traceback +import sys +from os.path import exists, join, dirname, abspath +from os import makedirs, environ +from future import standard_library + +from qiita_client import QiitaClient + +with standard_library.hooks(): + from configparser import ConfigParser + + +class BaseQiitaPlugin(object): + def __init__(self, name, ): + self.task_dict = {} + + def _register_command(self, command_name, function): + """Registers a command in the plugin + + `function` should be a callable that conforms to the signature: + function(qclient, job_id, job_parameters, output_dir) + where qclient is an instance of QiitaClient, job_id is a string with + the job identifier, job_parameters is a dictionary with the parameters + of the command and output_dir is a string with the output directory + + Parameters + ---------- + command_name : str + The command name + function : callable + The function that executed the command + + Raises + ------ + TypeError + If `function` is not callable + ValueError + If `function` does not accept 4 parameters + """ + # First make sure that `function` is callable + if not callable(function): + raise TypeError( + "Couldn't register command '%s': the provided function is not " + "callable (type: %s)" % (command_name, type(function))) + # `function` will be called with the following Parameters + # qclien, job_id, job_parameters, output_dir + # Make sure that `function` can receive 4 parameters + if function.__code__.co_argcount != 4: + raise ValueError( + "Couldn't register command '%s': the provided function does " + "not accept 4 parameters (number of parameters: %d)" + % (command_name, function.__code__.co_argcount)) + self.task_dict[command_name] = function + + def __call__(self, server_url, job_id, output_dir): + """Runs the plugin and executed the assigned task + + Parameters + ---------- + server_url : str + The url of the server + job_id : str + The job id + output_dir : str + The output directory + + Raises + ------ + RuntimeError + If there is a problem gathering the job information + """ + # Set up the Qiita Client + dflt_conf_fp = join(dirname(abspath(__file__)), 'support_files', + 'config_file.cfg') + conf_fp = environ.get('QP_TARGET_GENE_CONFIG_FP', dflt_conf_fp) + config = ConfigParser() + with open(conf_fp, 'U') as conf_file: + config.readfp(conf_file) + + qclient = QiitaClient(server_url, config.get('main', 'CLIENT_ID'), + config.get('main', 'CLIENT_SECRET'), + server_cert=config.get('main', 'SERVER_CERT')) + + # Request job information. If there is a problem retrieving the job + # information, the QiitaClient already raises an error + job_info = qclient.get_job_info(job_id) + # Starting the heartbeat + qclient.start_heartbeat(job_id) + # Execute the given task + task_name = job_info['command'] + task = self.task_dict[task_name] + + if not exists(output_dir): + makedirs(output_dir) + try: + success, artifacts_info, error_msg = task( + qclient, job_id, job_info['parameters'], output_dir) + except Exception: + exc_str = repr(traceback.format_exception(*sys.exc_info())) + error_msg = ("Error executing %s:\n%s" % (task_name, exc_str)) + success = False + artifacts_info = None + # The job completed + qclient.complete_job(job_id, success, error_msg=error_msg, + artifacts_info=artifacts_info) + + +class QiitaTypePlugin(BaseQiitaPlugin): + """Represents a Qiita Type Plugin + + Parameters + ---------- + validate_func : callable + The function used to validate artifacts + html_generator_func : callable + The function used to generate the HTML generator + + Notes + ----- + Both `validate_func` and `html_generator_func` should be a callable + that conforms to the signature: + function(qclient, job_id, job_parameters, output_dir) + where qclient is an instance of QiitaClient, job_id is a string with + the job identifier, job_parameters is a dictionary with the parameters + of the command and output_dir is a string with the output directory + """ + # List the available commands for a Qiita Type plugin + _valid_commands = {'Validate', 'Generate HTML summary'} + + def __init__(self, name, version, validate_func, html_generator_func): + super(QiitaTypePlugin, self).__init__() + + self._register_command('Validate', validate_func) + self._register_command('Generate HTML summary', html_generator_func) + + +class QiitaPlugin(BaseQiitaPlugin): + """Represents a Qiita Plugin""" + + def register_command(self, command_name, function): + """Registers a command in the plugin + + `function` should be a callable that conforms to the signature: + function(qclient, job_id, job_parameters, output_dir) + where qclient is an instance of QiitaClient, job_id is a string with + the job identifier, job_parameters is a dictionary with the parameters + of the command and output_dir is a string with the output directory + + Parameters + ---------- + command_name : str + The command name + function : callable + The function that executed the command + + Raises + ------ + TypeError + If `function` is not callable + ValueError + If `function` does not accept 4 parameters + """ + self._register_command(command_name, function) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py new file mode 100644 index 0000000..aef8d60 --- /dev/null +++ b/qiita_client/tests/test_plugin.py @@ -0,0 +1,22 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from unittest import TestCase, main + +from qiita_client import QiitaPlugin, QiitaTypePlugin + + +class QiitaPluginTest(TestCase): + pass + + +class QiitaTypePluginTest(TestCase): + pass + +if __name__ == '__main__': + main() From 151b20bc5ef41f3dba32a95a9781b8ae83ebdb14 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Mon, 26 Sep 2016 16:19:39 -0700 Subject: [PATCH 2/9] Adding command class --- qiita_client/__init__.py | 6 +- qiita_client/plugin.py | 156 +++++++++++++++++++----------- qiita_client/tests/test_plugin.py | 43 +++++++- 3 files changed, 146 insertions(+), 59 deletions(-) diff --git a/qiita_client/__init__.py b/qiita_client/__init__.py index 31c0752..53c61d5 100644 --- a/qiita_client/__init__.py +++ b/qiita_client/__init__.py @@ -9,8 +9,8 @@ from .exceptions import (QiitaClientError, NotFoundError, BadRequestError, ForbiddenError) from .qiita_client import QiitaClient, ArtifactInfo -from .plugin import QiitaPlugin, QiitaTypePlugin +from .plugin import QiitaCommand, QiitaPlugin, QiitaTypePlugin __all__ = ["QiitaClient", "QiitaClientError", "NotFoundError", - "BadRequestError", "ForbiddenError", "ArtifactInfo", "QiitaPlugin", - "QiitaTypePlugin"] + "BadRequestError", "ForbiddenError", "ArtifactInfo", "QiitaCommand", + "QiitaPlugin", "QiitaTypePlugin"] diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index ada341c..124c0b2 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -9,7 +9,7 @@ import traceback import sys from os.path import exists, join, dirname, abspath -from os import makedirs, environ +from os import makedirs from future import standard_library from qiita_client import QiitaClient @@ -18,47 +18,93 @@ from configparser import ConfigParser -class BaseQiitaPlugin(object): - def __init__(self, name, ): - self.task_dict = {} +class QiitaCommand(object): + """A plugin command - def _register_command(self, command_name, function): - """Registers a command in the plugin - - `function` should be a callable that conforms to the signature: - function(qclient, job_id, job_parameters, output_dir) + Parameters + ---------- + name : str + The command name + description : str + The command description + function : callable + The function that executes the command. Should be a callable that + conforms to the signature: + `function(qclient, job_id, job_parameters, output_dir)` where qclient is an instance of QiitaClient, job_id is a string with the job identifier, job_parameters is a dictionary with the parameters of the command and output_dir is a string with the output directory + required_parameters : dict of {str: (str, list of str)} + The required parameters of the command, keyed by parameter name. The + values should be a 2-tuple in which the first element is the parameter + type, and the second parameter is the list of subtypes (if applicable) + optional_parameters : dict of {str: (str, str)} + The optional parameters of the command, keyed by parameter name. The + values should be a 2-tuple in which the first element is the parameter + name, and the second paramater is the default value + default_parameter_sets : dict of {str: dict of {str: str}} + The default parameter sets of the command, keyed by parameter set name. + The values should be a dictionary in which keys are the parameter names + and values are the specific value for each parameter + + Raises + ------ + TypeError + If `function` is not callable + ValueError + If `function` does not accept 4 parameters + """ + def __init__(self, name, description, function, required_parameters, + optional_parameters, default_parameter_sets=None): + self.name = name + self.description = description - Parameters - ---------- - command_name : str - The command name - function : callable - The function that executed the command - - Raises - ------ - TypeError - If `function` is not callable - ValueError - If `function` does not accept 4 parameters - """ - # First make sure that `function` is callable + # Make sure that `function` is callable if not callable(function): raise TypeError( - "Couldn't register command '%s': the provided function is not " - "callable (type: %s)" % (command_name, type(function))) + "Couldn't create command '%s': the provided function is not " + "callable (type: %s)" % (name, type(function))) + # `function` will be called with the following Parameters - # qclien, job_id, job_parameters, output_dir + # qclient, job_id, job_parameters, output_dir # Make sure that `function` can receive 4 parameters if function.__code__.co_argcount != 4: raise ValueError( "Couldn't register command '%s': the provided function does " "not accept 4 parameters (number of parameters: %d)" - % (command_name, function.__code__.co_argcount)) - self.task_dict[command_name] = function + % (name, function.__code__.co_argcount)) + + self.function = function + self.required_parameters = required_parameters + self.optional_parameters = optional_parameters + self.default_parameter_sets = default_parameter_sets + + def __call__(self, qclient, server_url, job_id, output_dir): + return self.function(qclient, server_url, job_id, output_dir) + + +class BaseQiitaPlugin(object): + def __init__(self, name, version, description, conf_fp=None): + self.name = name + self.version = version + self.decription = description + self.conf_fp = conf_fp if conf_fp is not None else join( + dirname(abspath(__file__)), 'support_files', 'config_file.cfg') + self.task_dict = {} + + def _register_command(self, command): + """Registers a command in the plugin + + Parameters + ---------- + command: QiitaCommand + The command to be added to the plugin + """ + self.task_dict[command.name] = command + + # def _install(self, qclient): + # """Installs the plugin in Qiita""" + # for cmd in self.task_dict.values(): def __call__(self, server_url, job_id, output_dir): """Runs the plugin and executed the assigned task @@ -78,39 +124,39 @@ def __call__(self, server_url, job_id, output_dir): If there is a problem gathering the job information """ # Set up the Qiita Client - dflt_conf_fp = join(dirname(abspath(__file__)), 'support_files', - 'config_file.cfg') - conf_fp = environ.get('QP_TARGET_GENE_CONFIG_FP', dflt_conf_fp) config = ConfigParser() - with open(conf_fp, 'U') as conf_file: + with open(self.onf_fp, 'U') as conf_file: config.readfp(conf_file) qclient = QiitaClient(server_url, config.get('main', 'CLIENT_ID'), config.get('main', 'CLIENT_SECRET'), server_cert=config.get('main', 'SERVER_CERT')) - # Request job information. If there is a problem retrieving the job - # information, the QiitaClient already raises an error - job_info = qclient.get_job_info(job_id) - # Starting the heartbeat - qclient.start_heartbeat(job_id) - # Execute the given task - task_name = job_info['command'] - task = self.task_dict[task_name] - - if not exists(output_dir): - makedirs(output_dir) - try: - success, artifacts_info, error_msg = task( - qclient, job_id, job_info['parameters'], output_dir) - except Exception: - exc_str = repr(traceback.format_exception(*sys.exc_info())) - error_msg = ("Error executing %s:\n%s" % (task_name, exc_str)) - success = False - artifacts_info = None - # The job completed - qclient.complete_job(job_id, success, error_msg=error_msg, - artifacts_info=artifacts_info) + if job_id == 'register': + self._install(qclient) + else: + # Request job information. If there is a problem retrieving the job + # information, the QiitaClient already raises an error + job_info = qclient.get_job_info(job_id) + # Starting the heartbeat + qclient.start_heartbeat(job_id) + # Execute the given task + task_name = job_info['command'] + task = self.task_dict[task_name] + + if not exists(output_dir): + makedirs(output_dir) + try: + success, artifacts_info, error_msg = task( + qclient, job_id, job_info['parameters'], output_dir) + except Exception: + exc_str = repr(traceback.format_exception(*sys.exc_info())) + error_msg = ("Error executing %s:\n%s" % (task_name, exc_str)) + success = False + artifacts_info = None + # The job completed + qclient.complete_job(job_id, success, error_msg=error_msg, + artifacts_info=artifacts_info) class QiitaTypePlugin(BaseQiitaPlugin): diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index aef8d60..c651633 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -8,7 +8,48 @@ from unittest import TestCase, main -from qiita_client import QiitaPlugin, QiitaTypePlugin +from qiita_client import QiitaPlugin, QiitaTypePlugin, QiitaCommand + + +class QiitaCommandTest(TestCase): + def setUp(self): + self.exp_req = {'p1': ('artifact', ['FASTQ'])} + self.exp_opt = {'p2': ('boolean', 'False'), + 'p3': ('string', 'somestring')} + self.exp_dflt = {'dflt1': {'p2': 'True', 'p3': 'anotherstring'}} + + def test_init(self): + # Create a test function + def func(a, b, c, d): + return 42 + + obs = QiitaCommand("Test cmd", "Some description", func, self.exp_req, + self.exp_opt, self.exp_dflt) + self.assertEqual(obs.name, "Test cmd") + self.assertEqual(obs.description, "Some description") + self.assertEqual(obs.function, func) + self.assertEqual(obs.required_parameters, self.exp_req) + self.assertEqual(obs.optional_parameters, self.exp_opt) + self.assertEqual(obs.default_parameter_sets, self.exp_dflt) + + with self.assertRaises(TypeError): + QiitaCommand("Name", "Desc", "func", self.exp_req, self.exp_opt, + self.exp_dflt) + + def func(a, b, c): + return 42 + + with self.assertRaises(ValueError): + QiitaCommand("Name", "Desc", func, self.exp_req, self.exp_opt, + self.exp_dflt) + + def test_call(self): + def func(a, b, c, d): + return 42 + + obs = QiitaCommand("Test cmd", "Some description", func, self.exp_req, + self.exp_opt, self.exp_dflt) + self.assertEqual(obs('a', 'b', 'c', 'd'), 42) class QiitaPluginTest(TestCase): From f431fc3cca497f2c3688ca54c0913f699f83a148 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 27 Sep 2016 11:23:42 -0700 Subject: [PATCH 3/9] Type plugin successfully installed --- qiita_client/__init__.py | 5 +- qiita_client/plugin.py | 162 +++++++++++++++++++++--- qiita_client/testing.py | 29 +++++ qiita_client/tests/test_plugin.py | 116 ++++++++++++++++- qiita_client/tests/test_qiita_client.py | 11 +- 5 files changed, 289 insertions(+), 34 deletions(-) create mode 100644 qiita_client/testing.py diff --git a/qiita_client/__init__.py b/qiita_client/__init__.py index 53c61d5..a582196 100644 --- a/qiita_client/__init__.py +++ b/qiita_client/__init__.py @@ -9,8 +9,9 @@ from .exceptions import (QiitaClientError, NotFoundError, BadRequestError, ForbiddenError) from .qiita_client import QiitaClient, ArtifactInfo -from .plugin import QiitaCommand, QiitaPlugin, QiitaTypePlugin +from .plugin import (QiitaCommand, QiitaPlugin, QiitaTypePlugin, + QiitaArtifactType) __all__ = ["QiitaClient", "QiitaClientError", "NotFoundError", "BadRequestError", "ForbiddenError", "ArtifactInfo", "QiitaCommand", - "QiitaPlugin", "QiitaTypePlugin"] + "QiitaPlugin", "QiitaTypePlugin", "QiitaArtifactType"] diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index 124c0b2..2373e81 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -8,9 +8,12 @@ import traceback import sys -from os.path import exists, join, dirname, abspath -from os import makedirs +from string import ascii_letters, digits +from random import SystemRandom +from os.path import exists, join, expanduser +from os import makedirs, environ from future import standard_library +from json import dumps from qiita_client import QiitaClient @@ -83,15 +86,61 @@ def __call__(self, qclient, server_url, job_id, output_dir): return self.function(qclient, server_url, job_id, output_dir) +class QiitaArtifactType(object): + """A Qiita artifact type + + Parameters + ---------- + name : str + The artifact type name + description : str + The artifact type description + can_be_submitted_to_ebi : bool + Whether the artifact type can be submitted to EBI or not + can_be_submitted_to_vamps : bool + Whether the artifact type can be submitted to VAMPS or not + filepath_types : list of (str, bool) + The list filepath types that the new artifact type supports, and + if they're required or not in an artifact instance of this type""" + def __init__(self, name, description, can_be_submitted_to_ebi, + can_be_submitted_to_vamps, filepath_types): + self.name = name + self.description = description + self.ebi = can_be_submitted_to_ebi + self.vamps = can_be_submitted_to_vamps + self.fp_types = filepath_types + + class BaseQiitaPlugin(object): - def __init__(self, name, version, description, conf_fp=None): + def __init__(self, name, version, description, publications=None): self.name = name self.version = version - self.decription = description - self.conf_fp = conf_fp if conf_fp is not None else join( - dirname(abspath(__file__)), 'support_files', 'config_file.cfg') + self.description = description + self.publications = dumps(publications) if publications else "" + + # Will hold the different commands self.task_dict = {} + # The configuration file + conf_dir = environ.get( + 'QIITA_PLUGINS_DIR', join(expanduser('~'), '.qiita_plugins')) + self.conf_fp = join(conf_dir, "%s_%s.conf" % (self.name, self.version)) + + def generate_config(self, env_script, start_script, server_cert=None): + """Generates the plugin configuration file""" + sr = SystemRandom() + chars = ascii_letters + digits + client_id = ''.join(sr.choice(chars) for i in range(50)) + client_secret = ''.join(sr.choice(chars) for i in range(255)) + + server_cert = server_cert if server_cert else "" + + with open(self.conf_fp, 'w') as f: + f.write(CONF_TEMPLATE % (self.name, self.version, self.description, + env_script, start_script, + self._plugin_type, self.publications, + server_cert, client_id, client_secret)) + def _register_command(self, command): """Registers a command in the plugin @@ -102,9 +151,31 @@ def _register_command(self, command): """ self.task_dict[command.name] = command - # def _install(self, qclient): - # """Installs the plugin in Qiita""" - # for cmd in self.task_dict.values(): + def _install(self, qclient): + """Installs the plugin in Qiita""" + # Get the command information from qiita + info = qclient.get('/qiita_db/plugins/%s/%s/' + % (self.name, self.version)) + + for cmd in self.task_dict.values(): + if cmd.name in info['commands']: + qclient.post('/qiita_db/plugins/%s/%s/commands/%s/activate/' + % (self.name, self.version, + cmd.name.replace(' ', "%20"))) + else: + req_params = { + k: v if v[0] != 'artifact' else ['artifact:%s' + % dumps(v[1]), None] + for k, v in cmd.required_parameters.items()} + + data = {'name': cmd.name, + 'description': cmd.description, + 'required_parameters': dumps(req_params), + 'optional_parameters': dumps(cmd.optional_parameters), + 'default_parameter_sets': dumps( + cmd.default_parameter_sets)} + qclient.post('/qiita_db/plugins/%s/%s/commands/' + % (self.name, self.version), data=data) def __call__(self, server_url, job_id, output_dir): """Runs the plugin and executed the assigned task @@ -125,12 +196,12 @@ def __call__(self, server_url, job_id, output_dir): """ # Set up the Qiita Client config = ConfigParser() - with open(self.onf_fp, 'U') as conf_file: + with open(self.conf_fp, 'U') as conf_file: config.readfp(conf_file) - qclient = QiitaClient(server_url, config.get('main', 'CLIENT_ID'), - config.get('main', 'CLIENT_SECRET'), - server_cert=config.get('main', 'SERVER_CERT')) + qclient = QiitaClient(server_url, config.get('oauth2', 'CLIENT_ID'), + config.get('oauth2', 'CLIENT_SECRET'), + server_cert=config.get('oauth2', 'SERVER_CERT')) if job_id == 'register': self._install(qclient) @@ -164,10 +235,18 @@ class QiitaTypePlugin(BaseQiitaPlugin): Parameters ---------- + name : str + The plugin name + version : str + The plugin version + description : str + The plugin description validate_func : callable The function used to validate artifacts html_generator_func : callable The function used to generate the HTML generator + artifact_types : list of QiitaArtifactType + The artifact types defined in this plugin Notes ----- @@ -178,19 +257,49 @@ class QiitaTypePlugin(BaseQiitaPlugin): the job identifier, job_parameters is a dictionary with the parameters of the command and output_dir is a string with the output directory """ - # List the available commands for a Qiita Type plugin - _valid_commands = {'Validate', 'Generate HTML summary'} + _plugin_type = "artifact definition" + + def __init__(self, name, version, description, validate_func, + html_generator_func, artifact_types, publications=None): + super(QiitaTypePlugin, self).__init__(name, version, description, + publications=publications) + + self.artifact_types = artifact_types + + val_cmd = QiitaCommand( + 'Validate', 'Validates a new artifact', validate_func, + {'template': ('prep_template', None), + 'files': ('string', None), + 'artifact_type': ('string', None)}, {}) + + self._register_command(val_cmd) - def __init__(self, name, version, validate_func, html_generator_func): - super(QiitaTypePlugin, self).__init__() + html_cmd = QiitaCommand( + 'Generate HTML summary', 'Generates the HTML summary', + html_generator_func, + {'input_data': ('artifact', + [a.name for a in self.artifact_types])}, {}) - self._register_command('Validate', validate_func) - self._register_command('Generate HTML summary', html_generator_func) + self._register_command(html_cmd) + + def _install(self, qclient): + """Installs the plugin in Qiita""" + for at in self.artifact_types: + data = {'type_name': at.name, + 'description': at.description, + 'can_be_submitted_to_ebi': at.ebi, + 'can_be_submitted_to_vamps': at.vamps, + 'filepath_types': dumps(at.fp_types)} + qclient.post('/qiita_db/artifacts/types/', data=data) + + super(QiitaTypePlugin, self)._install(qclient) class QiitaPlugin(BaseQiitaPlugin): """Represents a Qiita Plugin""" + _plugin_type = "artifact transformation" + def register_command(self, command_name, function): """Registers a command in the plugin @@ -215,3 +324,18 @@ def register_command(self, command_name, function): If `function` does not accept 4 parameters """ self._register_command(command_name, function) + + +CONF_TEMPLATE = """[main] +NAME = %s +VERSION = %s +DESCRIPTION = %s +ENVIRONMENT_SCRIPT = %s +START_SCRIPT = %s +PLUGIN_TYPE = %s +PUBLICATIONS = %s + +[oauth2] +SERVER_CERT = %s +CLIENT_ID = %s +CLIENT_SECRET = %s""" diff --git a/qiita_client/testing.py b/qiita_client/testing.py new file mode 100644 index 0000000..7acfd83 --- /dev/null +++ b/qiita_client/testing.py @@ -0,0 +1,29 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from unittest import TestCase +from os import environ + +from qiita_client import QiitaClient + + +class PluginTestCase(TestCase): + @classmethod + def setUpClass(cls): + cls.client_id = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4' + cls.client_secret = ('J7FfQ7CQdOxuKhQAf1eoGgBAE81Ns8Gu3EKaWFm3IO2JKh' + 'AmmCWZuabe0O5Mp28s1') + cls.server_cert = environ.get('QIITA_SERVER_CERT', None) + cls.qclient = QiitaClient("https://localhost:21174", cls.client_id, + cls.client_secret, + server_cert=cls.server_cert) + + @classmethod + def tearDownClass(cls): + # Reset the test database + cls.qclient.post("/apitest/reset/") diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index c651633..0cb9b9c 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -8,7 +8,13 @@ from unittest import TestCase, main -from qiita_client import QiitaPlugin, QiitaTypePlugin, QiitaCommand +from os.path import isdir, join, exists, basename +from os import remove +from shutil import rmtree + +from qiita_client.testing import PluginTestCase +from qiita_client import (QiitaPlugin, QiitaTypePlugin, QiitaCommand, + QiitaArtifactType) class QiitaCommandTest(TestCase): @@ -52,11 +58,113 @@ def func(a, b, c, d): self.assertEqual(obs('a', 'b', 'c', 'd'), 42) -class QiitaPluginTest(TestCase): - pass +class QiitaArtifactTypeTest(TestCase): + def test_init(self): + obs = QiitaArtifactType('Name', 'Description', False, True, + [('plain_text', False)]) + self.assertEqual(obs.name, 'Name') + self.assertEqual(obs.description, 'Description') + self.assertFalse(obs.ebi) + self.assertTrue(obs.vamps) + self.assertEqual(obs.fp_types, [('plain_text', False)]) + + +class QiitaTypePluginTest(PluginTestCase): + def setUp(self): + self.clean_up_fp = [] + + def tearDown(self): + for fp in self.clean_up_fp: + if exists(fp): + if isdir(fp): + rmtree(fp) + else: + remove(fp) + + def test_init(self): + def validate_func(a, b, c, d): + return 42 + + def html_generator_func(a, b, c, d): + return 42 + + atypes = [QiitaArtifactType('Name', 'Description', False, True, + [('plain_text', False)])] + obs = QiitaTypePlugin("NewPlugin", "1.0.0", "Description", + validate_func, html_generator_func, + atypes) + self.assertEqual(obs.name, "NewPlugin") + self.assertEqual(obs.version, "1.0.0") + self.assertEqual(obs.description, "Description") + self.assertItemsEqual(obs.task_dict.keys(), + ['Validate', 'Generate HTML summary']) + self.assertEqual(obs.task_dict['Validate'].function, validate_func) + self.assertEqual(obs.task_dict['Generate HTML summary'].function, + html_generator_func) + self.assertEqual(obs.artifact_types, atypes) + self.assertEqual(basename(obs.conf_fp), 'NewPlugin_1.0.0.conf') + + def test_generate_config(self): + def validate_func(a, b, c, d): + return 42 + + def html_generator_func(a, b, c, d): + return 42 + atypes = [QiitaArtifactType('Name', 'Description', False, True, + [('plain_text', False)])] + tester = QiitaTypePlugin("NewPlugin", "1.0.0", "Description", + validate_func, html_generator_func, atypes) + + tester.generate_config('env_script', 'start_script') + self.assertTrue(exists(tester.conf_fp)) + with open(tester.conf_fp, 'U') as f: + conf = f.readlines() + + exp_lines = ['[main]\n', + 'NAME = NewPlugin\n', + 'VERSION = 1.0.0\n', + 'DESCRIPTION = Description\n', + 'ENVIRONMENT_SCRIPT = env_script\n', + 'START_SCRIPT = start_script\n', + 'PLUGIN_TYPE = artifact definition\n', + 'PUBLICATIONS = \n', + '\n', + '[oauth2]\n', + 'SERVER_CERT = \n'] + # We will test the last 2 lines independently since they're variable + # in each test run + self.assertEqual(conf[:-2], exp_lines) + self.assertTrue(conf[-2].startswith('CLIENT_ID = ')) + self.assertTrue(conf[-1].startswith('CLIENT_SECRET = ')) + + def test_call(self): + def validate_func(a, b, c, d): + return 42 + + def html_generator_func(a, b, c, d): + return 42 + + # Test the install procedure + atypes = [QiitaArtifactType('Name', 'Description', False, True, + [('plain_text', False)])] + tester = QiitaTypePlugin("NewPlugin", "1.0.0", "Description", + validate_func, html_generator_func, atypes) + + # Generate the config file for the new plugin + tester.generate_config('env_script', 'start_script', + server_cert=self.server_cert) + # Ask Qiita to reload the plugins + self.qclient.post('/apitest/reload_plugins/') + + # Install the current plugin + tester("https://localhost:21174", 'register', 'ignored') + + # Check that it has been installed + obs = self.qclient.get('/qiita_db/plugins/NewPlugin/1.0.0/') + self.assertEqual(obs['name'], 'NewPlugin') -class QiitaTypePluginTest(TestCase): +class QiitaPluginTest(PluginTestCase): pass if __name__ == '__main__': diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 5d19b6b..cdad7d2 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -14,6 +14,7 @@ from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) +from qiita_client.testing import PluginTestCase from qiita_client.exceptions import BadRequestError CLIENT_ID = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4' @@ -66,15 +67,7 @@ def test_format_payload_error(self): self.assertEqual(obs, exp) -class QiitaClientTests(TestCase): - @classmethod - def tearDownClass(cls): - # Reset the test database - server_cert = environ.get('QIITA_SERVER_CERT', None) - qclient = QiitaClient("https://localhost:21174", CLIENT_ID, - CLIENT_SECRET, server_cert=server_cert) - qclient.post("/apitest/reset/") - +class QiitaClientTests(PluginTestCase): def setUp(self): self.server_cert = environ.get('QIITA_SERVER_CERT', None) self.tester = QiitaClient("https://localhost:21174", CLIENT_ID, From 905eada87e715ac5bedc711c78747e96a6e7c513 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 27 Sep 2016 12:20:43 -0700 Subject: [PATCH 4/9] Finishing up code --- qiita_client/plugin.py | 33 ++++++---------- qiita_client/tests/test_plugin.py | 62 +++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index 2373e81..354479a 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -45,6 +45,9 @@ class QiitaCommand(object): The optional parameters of the command, keyed by parameter name. The values should be a 2-tuple in which the first element is the parameter name, and the second paramater is the default value + outputs : dict of {str: str} + The description of the outputs that this command generated. The + format is: {output_name: artifact_type} default_parameter_sets : dict of {str: dict of {str: str}} The default parameter sets of the command, keyed by parameter set name. The values should be a dictionary in which keys are the parameter names @@ -58,7 +61,7 @@ class QiitaCommand(object): If `function` does not accept 4 parameters """ def __init__(self, name, description, function, required_parameters, - optional_parameters, default_parameter_sets=None): + optional_parameters, outputs, default_parameter_sets=None): self.name = name self.description = description @@ -81,6 +84,7 @@ def __init__(self, name, description, function, required_parameters, self.required_parameters = required_parameters self.optional_parameters = optional_parameters self.default_parameter_sets = default_parameter_sets + self.outputs = outputs def __call__(self, qclient, server_url, job_id, output_dir): return self.function(qclient, server_url, job_id, output_dir) @@ -270,7 +274,7 @@ def __init__(self, name, version, description, validate_func, 'Validate', 'Validates a new artifact', validate_func, {'template': ('prep_template', None), 'files': ('string', None), - 'artifact_type': ('string', None)}, {}) + 'artifact_type': ('string', None)}, {}, None) self._register_command(val_cmd) @@ -278,7 +282,7 @@ def __init__(self, name, version, description, validate_func, 'Generate HTML summary', 'Generates the HTML summary', html_generator_func, {'input_data': ('artifact', - [a.name for a in self.artifact_types])}, {}) + [a.name for a in self.artifact_types])}, {}, None) self._register_command(html_cmd) @@ -300,30 +304,15 @@ class QiitaPlugin(BaseQiitaPlugin): _plugin_type = "artifact transformation" - def register_command(self, command_name, function): + def register_command(self, command): """Registers a command in the plugin - `function` should be a callable that conforms to the signature: - function(qclient, job_id, job_parameters, output_dir) - where qclient is an instance of QiitaClient, job_id is a string with - the job identifier, job_parameters is a dictionary with the parameters - of the command and output_dir is a string with the output directory - Parameters ---------- - command_name : str - The command name - function : callable - The function that executed the command - - Raises - ------ - TypeError - If `function` is not callable - ValueError - If `function` does not accept 4 parameters + command: QiitaCommand + The command to be added to the plugin """ - self._register_command(command_name, function) + self._register_command(command) CONF_TEMPLATE = """[main] diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index 0cb9b9c..4495e15 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -7,14 +7,15 @@ # ----------------------------------------------------------------------------- from unittest import TestCase, main - -from os.path import isdir, join, exists, basename +from os.path import isdir, exists, basename, join from os import remove from shutil import rmtree +from json import dumps +from tempfile import mkdtemp from qiita_client.testing import PluginTestCase from qiita_client import (QiitaPlugin, QiitaTypePlugin, QiitaCommand, - QiitaArtifactType) + QiitaArtifactType, ArtifactInfo) class QiitaCommandTest(TestCase): @@ -23,6 +24,7 @@ def setUp(self): self.exp_opt = {'p2': ('boolean', 'False'), 'p3': ('string', 'somestring')} self.exp_dflt = {'dflt1': {'p2': 'True', 'p3': 'anotherstring'}} + self.exp_out = {'out1': 'BIOM'} def test_init(self): # Create a test function @@ -30,31 +32,32 @@ def func(a, b, c, d): return 42 obs = QiitaCommand("Test cmd", "Some description", func, self.exp_req, - self.exp_opt, self.exp_dflt) + self.exp_opt, self.exp_out, self.exp_dflt) self.assertEqual(obs.name, "Test cmd") self.assertEqual(obs.description, "Some description") self.assertEqual(obs.function, func) self.assertEqual(obs.required_parameters, self.exp_req) self.assertEqual(obs.optional_parameters, self.exp_opt) + self.assertEqual(obs.outputs, self.exp_out) self.assertEqual(obs.default_parameter_sets, self.exp_dflt) with self.assertRaises(TypeError): QiitaCommand("Name", "Desc", "func", self.exp_req, self.exp_opt, - self.exp_dflt) + self.exp_out, self.exp_dflt) def func(a, b, c): return 42 with self.assertRaises(ValueError): QiitaCommand("Name", "Desc", func, self.exp_req, self.exp_opt, - self.exp_dflt) + self.exp_out, self.exp_dflt) def test_call(self): def func(a, b, c, d): return 42 obs = QiitaCommand("Test cmd", "Some description", func, self.exp_req, - self.exp_opt, self.exp_dflt) + self.exp_opt, self.exp_out, self.exp_dflt) self.assertEqual(obs('a', 'b', 'c', 'd'), 42) @@ -162,10 +165,53 @@ def html_generator_func(a, b, c, d): # Check that it has been installed obs = self.qclient.get('/qiita_db/plugins/NewPlugin/1.0.0/') self.assertEqual(obs['name'], 'NewPlugin') + self.assertEqual(obs['version'], '1.0.0') class QiitaPluginTest(PluginTestCase): - pass + # Most of the functionility is being tested in the previous + # class. Here we are going to test that we can actually execute a job + def setUp(self): + self.outdir = mkdtemp() + + def tearDown(self): + rmtree(self.outdir) + + def test_call(self): + def func(qclient, job_id, job_params, working_dir): + fp = join(working_dir, 'test.fastq') + with open(fp, 'w') as f: + f.write('') + res = ArtifactInfo('out1', 'Demultiplexed', + [[fp, 'preprocessed_fastq']]) + return True, "", [res] + + tester = QiitaPlugin("NewPlugin", "0.0.1", "description") + cmd = QiitaCommand("NewCmd", "Desc", func, + {'p1': ('artifact', ['FASTQ'])}, + {'p2': ('string', 'dflt')}, + {'out1': 'Demultiplexed'}) + tester.register_command(cmd) + + tester.generate_config('env_script', 'start_script', + server_cert=self.server_cert) + self.qclient.post('/apitest/reload_plugins/') + tester("https://localhost:21174", 'register', 'ignored') + + obs = self.qclient.get('/qiita_db/plugins/NewPlugin/0.0.1/') + self.assertEqual(obs['name'], 'NewPlugin') + self.assertEqual(obs['version'], '0.0.1') + self.assertEqual(obs['commands'], ['NewCmd']) + + # Create a new job + data = {'command': dumps(['NewPlugin', '0.0.1', 'NewCmd']), + 'parameters': dumps({'p1': '1', 'p2': 'a'}), + 'status': 'queued'} + job_id = self.qclient.post('/apitest/processing_job/', + data=data)['job'] + tester("https://localhost:21174", job_id, self.outdir) + obs = self.qclient.get_job_info(job_id) + self.assertEqual(obs['status'], 'success') if __name__ == '__main__': main() From 01178f730ac5bed0ba5a3acfd9453b461f1e7ffb Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 27 Sep 2016 12:21:24 -0700 Subject: [PATCH 5/9] Updating travis to install the new Qiita branch --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 586bb41..589e47a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: - pip install sphinx sphinx-bootstrap-theme coveralls ipython[all]==2.4.1 # Installing the simplify REST api branch - change once it has been merged to master # - pip install https://github.com/biocore/qiita/archive/master.zip - - pip install https://github.com/biocore/qiita/archive/simplify-restAPI.zip + - pip install https://github.com/biocore/qiita/archive/plugin-installation.zip - ipython profile create qiita-general --parallel - qiita-env start_cluster qiita-general - qiita-env make --no-load-ontologies From 0ba3041d47dd777b748142e98938a6fc0bfc73d4 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 27 Sep 2016 12:39:31 -0700 Subject: [PATCH 6/9] Adding future to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d26023..e85a992 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,6 @@ test_suite='nose.collector', packages=['qiita_client'], extras_require={'test': ["nose >= 0.10.1", "pep8"]}, - install_requires=['click >= 3.3', 'requests'], + install_requires=['click >= 3.3', 'requests', 'future'], classifiers=classifiers ) From 1723b2cb3737e165ea8c574e4c281bd330dfe301 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 27 Sep 2016 12:54:49 -0700 Subject: [PATCH 7/9] Fixing py3 compat --- qiita_client/tests/test_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index 4495e15..18d0212 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -99,8 +99,8 @@ def html_generator_func(a, b, c, d): self.assertEqual(obs.name, "NewPlugin") self.assertEqual(obs.version, "1.0.0") self.assertEqual(obs.description, "Description") - self.assertItemsEqual(obs.task_dict.keys(), - ['Validate', 'Generate HTML summary']) + self.assertEqual(set(obs.task_dict.keys()), + {'Validate', 'Generate HTML summary'}) self.assertEqual(obs.task_dict['Validate'].function, validate_func) self.assertEqual(obs.task_dict['Generate HTML summary'].function, html_generator_func) From ca7d9aa0ac502d18da1211376a6058e631942702 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 27 Sep 2016 16:50:36 -0700 Subject: [PATCH 8/9] Fixing failing test --- qiita_client/tests/test_qiita_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index cdad7d2..61d052b 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -187,7 +187,7 @@ def test_complete_job(self): # Create a new job data = { 'user': 'demo@microbio.me', - 'command': 3, + 'command': dumps(['QIIME', '1.9.1', 'Pick closed-reference OTUs']), 'status': 'running', 'parameters': dumps({"reference": 1, "sortmerna_e_value": 1, From 2f45c38042165f05f08c9a8c68143f45a8b4491f Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 27 Sep 2016 22:46:27 -0700 Subject: [PATCH 9/9] Addressing @wasade's comments --- qiita_client/plugin.py | 46 +++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index 354479a..03c23e4 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -14,6 +14,7 @@ from os import makedirs, environ from future import standard_library from json import dumps +import urllib from qiita_client import QiitaClient @@ -33,10 +34,14 @@ class QiitaCommand(object): function : callable The function that executes the command. Should be a callable that conforms to the signature: - `function(qclient, job_id, job_parameters, output_dir)` + `(bool, str, [ArtifactInfo] = function(qclient, job_id, job_parameters, + output_dir)` where qclient is an instance of QiitaClient, job_id is a string with the job identifier, job_parameters is a dictionary with the parameters - of the command and output_dir is a string with the output directory + of the command and output_dir is a string with the output directory. + The function should return a boolean indicating if the command was + executed successfully or not, a string containing a message in case + of error, and a list of ArtifactInfo objects in case of success. required_parameters : dict of {str: (str, list of str)} The required parameters of the command, keyed by parameter name. The values should be a 2-tuple in which the first element is the parameter @@ -131,7 +136,20 @@ def __init__(self, name, version, description, publications=None): self.conf_fp = join(conf_dir, "%s_%s.conf" % (self.name, self.version)) def generate_config(self, env_script, start_script, server_cert=None): - """Generates the plugin configuration file""" + """Generates the plugin configuration file + + Parameters + ---------- + env_script : str + The CLI call used to load the environment in which the plugin is + installed + start_script : str + The script used to start the plugin + server_cert : str, optional + If the Qiita server used does not have a valid certificate, the + path to the Qiita certificate so the plugin can connect over + HTTPS to it + """ sr = SystemRandom() chars = ascii_letters + digits client_id = ''.join(sr.choice(chars) for i in range(50)) @@ -155,8 +173,8 @@ def _register_command(self, command): """ self.task_dict[command.name] = command - def _install(self, qclient): - """Installs the plugin in Qiita""" + def _register(self, qclient): + """Registers the plugin information in Qiita""" # Get the command information from qiita info = qclient.get('/qiita_db/plugins/%s/%s/' % (self.name, self.version)) @@ -165,7 +183,7 @@ def _install(self, qclient): if cmd.name in info['commands']: qclient.post('/qiita_db/plugins/%s/%s/commands/%s/activate/' % (self.name, self.version, - cmd.name.replace(' ', "%20"))) + urllib.parse.quote(cmd.name))) else: req_params = { k: v if v[0] != 'artifact' else ['artifact:%s' @@ -208,7 +226,7 @@ def __call__(self, server_url, job_id, output_dir): server_cert=config.get('oauth2', 'SERVER_CERT')) if job_id == 'register': - self._install(qclient) + self._register(qclient) else: # Request job information. If there is a problem retrieving the job # information, the QiitaClient already raises an error @@ -256,10 +274,14 @@ class QiitaTypePlugin(BaseQiitaPlugin): ----- Both `validate_func` and `html_generator_func` should be a callable that conforms to the signature: - function(qclient, job_id, job_parameters, output_dir) + `(bool, str, [ArtifactInfo] = function(qclient, job_id, job_parameters, + output_dir)` where qclient is an instance of QiitaClient, job_id is a string with the job identifier, job_parameters is a dictionary with the parameters - of the command and output_dir is a string with the output directory + of the command and output_dir is a string with the output directory. + The function should return a boolean indicating if the command was + executed successfully or not, a string containing a message in case + of error, and a list of ArtifactInfo objects in case of success. """ _plugin_type = "artifact definition" @@ -286,8 +308,8 @@ def __init__(self, name, version, description, validate_func, self._register_command(html_cmd) - def _install(self, qclient): - """Installs the plugin in Qiita""" + def _register(self, qclient): + """Registers the plugin information in Qiita""" for at in self.artifact_types: data = {'type_name': at.name, 'description': at.description, @@ -296,7 +318,7 @@ def _install(self, qclient): 'filepath_types': dumps(at.fp_types)} qclient.post('/qiita_db/artifacts/types/', data=data) - super(QiitaTypePlugin, self)._install(qclient) + super(QiitaTypePlugin, self)._register(qclient) class QiitaPlugin(BaseQiitaPlugin):