diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..7a2486b41 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "planemo/xml/xsd/tool"] + path = planemo/xml/xsd/tool + url = https://github.com/JeanFred/Galaxy-XSD diff --git a/planemo/commands/cmd_lint.py b/planemo/commands/cmd_lint.py index cae1d8378..769b9b9f6 100644 --- a/planemo/commands/cmd_lint.py +++ b/planemo/commands/cmd_lint.py @@ -1,55 +1,24 @@ import sys -import traceback + import click from planemo.cli import pass_context -from planemo.io import info -from planemo.io import error from planemo import options -from galaxy.tools.loader_directory import load_tool_elements_from_path -from galaxy.tools.lint import lint_xml - -SKIP_XML_MESSAGE = "Skipping XML file - does not appear to be a tool %s." -LINTING_TOOL_MESSAGE = "Linting tool %s" +from planemo.tool_lint import build_lint_args +from planemo.tool_lint import lint_tools_on_path @click.command('lint') @options.optional_tools_arg() -@click.option( - '--report_level', - type=click.Choice(['all', 'warn', 'error']), - default="all" -) -@click.option( - '--fail_level', - type=click.Choice(['warn', 'error']), - default="warn" -) +@options.report_level_option() +@options.fail_level_option() +@options.lint_xsd_option() @pass_context -def cli(ctx, path, report_level="all", fail_level="warn"): +def cli(ctx, path, **kwds): """Check specified tool(s) for common errors and adherence to best practices. """ - exit = 0 - lint_args = dict(level=report_level, fail_level=fail_level) - tools = load_tool_elements_from_path(path, load_exception_handler) - valid_tools = 0 - for (tool_path, tool_xml) in tools: - if tool_xml.getroot().tag != "tool": - if ctx.verbose: - info(SKIP_XML_MESSAGE % tool_path) - continue - info("Linting tool %s" % tool_path) - if not lint_xml(tool_xml, **lint_args): - exit = 1 - else: - valid_tools += 1 - if exit == 0 and valid_tools == 0: - exit = 2 + lint_args = build_lint_args(**kwds) + exit = lint_tools_on_path(ctx, path, lint_args) sys.exit(exit) - - -def load_exception_handler(path, exc_info): - error("Error loading tool with path %s" % path) - traceback.print_exception(*exc_info, limit=1, file=sys.stderr) diff --git a/planemo/commands/cmd_shed_lint.py b/planemo/commands/cmd_shed_lint.py new file mode 100644 index 000000000..aed5b2e45 --- /dev/null +++ b/planemo/commands/cmd_shed_lint.py @@ -0,0 +1,34 @@ +import click +import sys + +from planemo.cli import pass_context +from planemo import options +from planemo import shed +from planemo import shed_lint + + +@click.command('shed_lint') +@options.optional_project_arg(exists=True) +@options.report_level_option() +@options.fail_level_option() +@options.click.option( + '--tools', + is_flag=True, + default=False, + help=("Lint tools discovered in the process of linting repositories.") +) +@options.lint_xsd_option() +@options.recursive_shed_option() +@pass_context +def cli(ctx, path, recursive=False, **kwds): + """Check a Tool Shed repository for common problems. + """ + def lint(path): + return shed_lint.lint_repository(ctx, path, **kwds) + + if recursive: + exit_code = shed.for_each_repository(lint, path) + else: + exit_code = lint(path) + + sys.exit(exit_code) diff --git a/planemo/commands/cmd_shed_upload.py b/planemo/commands/cmd_shed_upload.py index 375214e12..da69d29f7 100644 --- a/planemo/commands/cmd_shed_upload.py +++ b/planemo/commands/cmd_shed_upload.py @@ -10,8 +10,6 @@ from planemo.io import shell import json -import fnmatch -import os tar_path = click.Path( exists=True, @@ -54,15 +52,15 @@ is_flag=True, default=False ) -@click.option( - '-r', '--recursive', - is_flag=True, - help="Recursively search for repositories to publish to a tool shed", -) +@options.recursive_shed_option() @pass_context def cli(ctx, path, **kwds): """Handle possible recursion through paths for uploading files to a toolshed """ + + def upload(path): + return __handle_upload(ctx, **kwds) + if kwds['recursive']: if kwds['name'] is not None: error("--name is incompatible with --recursive") @@ -71,17 +69,9 @@ def cli(ctx, path, **kwds): error("--tar is incompatible with --recursive") return -1 - ret_codes = [] - for base_path, dirnames, filenames in os.walk(path): - for filename in fnmatch.filter(filenames, '.shed.yml'): - ret_codes.append( - __handle_upload(ctx, base_path, **kwds) - ) - # "Good" returns are Nones, everything else is a -1 and should be - # passed upwards. - return None if all(x is None for x in ret_codes) else -1 + return shed.for_each_repository(upload, path) else: - return __handle_upload(ctx, path, **kwds) + return upload(path) def __handle_upload(ctx, path, **kwds): @@ -121,6 +111,7 @@ def __handle_upload(ctx, path, **kwds): error(e2.read()) return -1 info("Repository %s updated successfully." % path) + return 0 def __find_repository(ctx, tsi, path, **kwds): diff --git a/planemo/galaxy_test.py b/planemo/galaxy_test.py index 8655ece54..0f2bc045b 100644 --- a/planemo/galaxy_test.py +++ b/planemo/galaxy_test.py @@ -1,5 +1,6 @@ """ Utilities for reasoning about Galaxy test results. """ +from __future__ import absolute_import import json import xml.etree.ElementTree as ET diff --git a/planemo/lint.py b/planemo/lint.py new file mode 100644 index 000000000..d36760201 --- /dev/null +++ b/planemo/lint.py @@ -0,0 +1,15 @@ +import os + +from planemo.xml import validation + + +def lint_xsd(lint_ctx, schema_path, path): + name = os.path.basename(path) + validator = validation.get_validator(require=True) + validation_result = validator.validate(schema_path, path) + if not validation_result.passed: + msg = "Invalid %s found. Errors [%s]" + msg = msg % (name, validation_result.output) + lint_ctx.error(msg) + else: + lint_ctx.info("%s found and appears to be valid XML" % name) diff --git a/planemo/linters/__init__.py b/planemo/linters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/planemo/linters/xsd.py b/planemo/linters/xsd.py new file mode 100644 index 000000000..4b1689e93 --- /dev/null +++ b/planemo/linters/xsd.py @@ -0,0 +1,30 @@ +""" Tool linting module that lints Galaxy tool against experimental XSD. +""" +import copy +import os +import tempfile + +from planemo.xml import XSDS_PATH +import planemo.lint + +TOOL_XSD = os.path.join(XSDS_PATH, "tool", "galaxy.xsd") + + +def lint_tool_xsd(root, lint_ctx): + """ Write a temp file out and lint it. + """ + with tempfile.NamedTemporaryFile() as tf: + _clean_root(root).write(tf.name) + planemo.lint.lint_xsd(lint_ctx, TOOL_XSD, tf.name) + + +def _clean_root(root): + """ XSD assumes macros have been expanded, so remove them. + """ + clean_root = copy.deepcopy(root) + to_remove = [] + for macros_el in clean_root.findall("macros"): + to_remove.append(macros_el) + for macros_el in to_remove: + clean_root.getroot().remove(macros_el) + return clean_root diff --git a/planemo/options.py b/planemo/options.py index dca77e5f0..671ff8a65 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -249,3 +249,38 @@ def shed_password_option(): help="Password for Tool Shed auth (required unless shed_key is " "specified)." ) + + +def lint_xsd_option(): + return click.option( + '--xsd', + is_flag=True, + default=False, + help=("Include experimental tool XSD validation in linting " + "process (requires xmllint on PATH or lxml installed).") + ) + + +def report_level_option(): + return click.option( + '--report_level', + type=click.Choice(['all', 'warn', 'error']), + default="all", + ) + + +def fail_level_option(): + return click.option( + '--fail_level', + type=click.Choice(['warn', 'error']), + default="warn" + ) + + +def recursive_shed_option(): + return click.option( + '-r', + '--recursive', + is_flag=True, + help="Recursively perform command for nested repository directories.", + ) diff --git a/planemo/shed.py b/planemo/shed.py index f1865a9d8..6c8042b62 100644 --- a/planemo/shed.py +++ b/planemo/shed.py @@ -1,8 +1,10 @@ +import fnmatch +import glob import os -from tempfile import mkstemp import tarfile +from tempfile import mkstemp + import yaml -import glob try: from bioblend import toolshed @@ -192,6 +194,24 @@ def build_tarball(tool_path): return temp_path +def walk_repositories(path): + """ Recurse through directories and find effective repositories. """ + for base_path, dirnames, filenames in os.walk(path): + for filename in fnmatch.filter(filenames, '.shed.yml'): + yield base_path + + +def for_each_repository(function, path): + ret_codes = [] + for base_path in walk_repositories(path): + ret_codes.append( + function(base_path) + ) + # "Good" returns are Nones, everything else is a -1 and should be + # passed upwards. + return 0 if all((not x) for x in ret_codes) else -1 + + def username(tsi): user = _user(tsi) return user["username"] diff --git a/planemo/shed_lint.py b/planemo/shed_lint.py new file mode 100644 index 000000000..a274e0a36 --- /dev/null +++ b/planemo/shed_lint.py @@ -0,0 +1,87 @@ +import os +import yaml +from galaxy.tools.lint import LintContext +from planemo.lint import lint_xsd +from planemo.tool_lint import ( + build_lint_args, + yield_tool_xmls, +) +from planemo.xml import XSDS_PATH + + +from planemo.io import info +from planemo.io import error + +from galaxy.tools.lint import lint_xml_with + +TOOL_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "tool_dependencies.xsd") +REPO_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "repository_dependencies.xsd") + + +def lint_repository(ctx, path, **kwds): + info("Linting repository %s" % path) + lint_args = build_lint_args(**kwds) + lint_ctx = LintContext(lint_args["level"]) + lint_ctx.lint( + "tool_dependencies", + lint_tool_dependencies, + path, + ) + lint_ctx.lint( + "repository_dependencies", + lint_repository_dependencies, + path, + ) + lint_ctx.lint( + "shed_yaml", + lint_shed_yaml, + path, + ) + if kwds["tools"]: + for (tool_path, tool_xml) in yield_tool_xmls(ctx, path): + info("+Linting tool %s" % tool_path) + lint_xml_with( + lint_ctx, + tool_xml, + extra_modules=lint_args["extra_modules"] + ) + failed = lint_ctx.failed(lint_args["fail_level"]) + if failed: + error("Failed linting") + return 1 if failed else 0 + + +def lint_tool_dependencies(path, lint_ctx): + tool_dependencies = os.path.join(path, "tool_dependencies.xml") + if not os.path.exists(tool_dependencies): + lint_ctx.info("No tool_dependencies.xml, skipping.") + return + lint_xsd(lint_ctx, TOOL_DEPENDENCIES_XSD, tool_dependencies) + + +def lint_repository_dependencies(path, lint_ctx): + repo_dependencies = os.path.join(path, "repository_dependencies.xml") + if not os.path.exists(repo_dependencies): + lint_ctx.info("No repository_dependencies.xml, skipping.") + return + lint_xsd(lint_ctx, REPO_DEPENDENCIES_XSD, repo_dependencies) + + +def lint_shed_yaml(path, lint_ctx): + shed_yaml = os.path.join(path, ".shed.yml") + if not os.path.exists(shed_yaml): + lint_ctx.info("No .shed.yml file found, skipping.") + return + try: + shed_contents = yaml.load(open(shed_yaml, "r")) + except Exception as e: + lint_ctx.warn("Failed to parse .shed.yml file [%s]" % str(e)) + + warned = False + for required_key in ["owner", "name"]: + if required_key not in shed_contents: + lint_ctx.warn(".shed.yml did not contain key [%s]" % required_key) + warned = True + + if not warned: + lint_ctx.info(".shed.yml found and appears to be valid YAML.") diff --git a/planemo/tool_lint.py b/planemo/tool_lint.py new file mode 100644 index 000000000..fca04d03e --- /dev/null +++ b/planemo/tool_lint.py @@ -0,0 +1,72 @@ +import os +import sys +import traceback + +from planemo.io import info +from planemo.io import error + +import planemo.linters.xsd + +from galaxy.tools.loader_directory import load_tool_elements_from_path +from galaxy.tools.lint import lint_xml + +SKIP_XML_MESSAGE = "Skipping XML file - does not appear to be a tool %s." +LINTING_TOOL_MESSAGE = "Linting tool %s" +SHED_FILES = ["tool_dependencies.xml", "repository_dependencies.xml"] + + +def lint_tools_on_path(ctx, path, lint_args, assert_tools=True): + exit = 0 + valid_tools = 0 + for (tool_path, tool_xml) in yield_tool_xmls(ctx, path): + info("Linting tool %s" % tool_path) + if not lint_xml(tool_xml, **lint_args): + error("Failed linting") + exit = 1 + else: + valid_tools += 1 + if exit == 0 and valid_tools == 0 and assert_tools: + exit = 2 + return exit + + +def yield_tool_xmls(ctx, path): + tools = load_tool_elements_from_path(path, load_exception_handler) + for (tool_path, tool_xml) in tools: + if not _is_tool_xml(ctx, tool_path, tool_xml): + continue + yield (tool_path, tool_xml) + + +def build_lint_args(**kwds): + report_level = kwds.get("report_level", "all") + fail_level = kwds.get("fail_level", "warn") + lint_args = dict( + level=report_level, + fail_level=fail_level, + extra_modules=_lint_extra_modules(**kwds) + ) + return lint_args + + +def load_exception_handler(path, exc_info): + error("Error loading tool with path %s" % path) + traceback.print_exception(*exc_info, limit=1, file=sys.stderr) + + +def _lint_extra_modules(**kwds): + xsd = kwds.get("xsd", False) + if xsd: + return [planemo.linters.xsd] + else: + return [] + + +def _is_tool_xml(ctx, tool_path, tool_xml): + if os.path.basename(tool_path) in SHED_FILES: + return False + if tool_xml.getroot().tag != "tool": + if ctx.verbose: + info(SKIP_XML_MESSAGE % tool_path) + return False + return True diff --git a/planemo/xml/__init__.py b/planemo/xml/__init__.py new file mode 100644 index 000000000..585f56cb1 --- /dev/null +++ b/planemo/xml/__init__.py @@ -0,0 +1,6 @@ +import os + +XML_DIRECTORY = os.path.dirname(__file__) +XSDS_PATH = os.path.join(XML_DIRECTORY, "xsd") + +__all__ = [XSDS_PATH] diff --git a/planemo/xml/validation.py b/planemo/xml/validation.py new file mode 100644 index 000000000..311f30d1f --- /dev/null +++ b/planemo/xml/validation.py @@ -0,0 +1,81 @@ +import abc +from collections import namedtuple +import subprocess + +from galaxy.tools.deps.commands import which + +try: + from lxml import etree +except ImportError: + etree = None + +XMLLINT_COMMAND = "xmllint --noout --schema {0} {1} 2>&1" +INSTALL_VALIDATOR_MESSAGE = ("This feature requires an external dependency " + "to function, pleaes install xmllint (e.g 'brew " + "install libxml2' or 'apt-get install " + "libxml2-utils'.") + + +class XsdValidator(object): + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def validate(self, schema_path, target_path): + """ Validate ``target_path`` against ``schema_path``. + + :return type: ValidationResult + """ + + @abc.abstractmethod + def enabled(self): + """ Return True iff system has dependencies for this validator. + + :return type: bool + """ + +ValidationResult = namedtuple("ValidationResult", ["passed", "output"]) + + +class LxmlValidator(XsdValidator): + """ Validate XSD files using lxml library. """ + + def validate(self, schema_path, target_path): + try: + xsd_doc = etree.parse(schema_path) + xsd = etree.XMLSchema(xsd_doc) + xml = etree.parse(target_path) + passed = xsd.validate(xml) + return ValidationResult(passed, xsd.error_log) + except etree.XMLSyntaxError as e: + return ValidationResult(False, str(e)) + + def enabled(self): + return etree is not None + + +class XmllintValidator(XsdValidator): + """ Validate XSD files with the external tool xmllint. """ + + def validate(self, schema_path, target_path): + command = XMLLINT_COMMAND.format(schema_path, target_path) + p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + stdout, _ = p.communicate() + passed = p.returncode == 0 + return ValidationResult(passed, stdout) + + def enabled(self): + return bool(which("xmllint")) + + +VALIDATORS = [LxmlValidator(), XmllintValidator()] + + +def get_validator(require=True): + for validator in VALIDATORS: + if validator.enabled(): + return validator + + if require: + raise Exception(INSTALL_VALIDATOR_MESSAGE) + + return None diff --git a/planemo/xml/xsd/repository_dependencies.xsd b/planemo/xml/xsd/repository_dependencies.xsd new file mode 100644 index 000000000..4c4c6e5ff --- /dev/null +++ b/planemo/xml/xsd/repository_dependencies.xsd @@ -0,0 +1,60 @@ + + + + + Tool Shed repository_dependencies.xml Schema + Describes dependencies this repository depends on for things other then tool dependencies. + + + + + Collection of repositories. + + + + + + + Description of repositries. + + + + + + + Defines a specific revision of a repository on which this repository depends. If the toolshed is not defined, it will be automatically set to the local Tool Shed. If defined, the changeset_revision is the minimum required version. If the changeset_revision is not defined, it will be set to the latest installable changeset_revision for the repository defined by the name and owner. If either the toolshed or the changeset_revision is not defined, the repository_dependencies.xml file will automatically be altered (before it is committed in the changeset) to include the attributes and values just discussed. + + + + Repository name + + + + + Owner of repository + + + + + Tool Shed containing repository. + + + + + Changeset revision of dependent repository. + + + + + Prior installation required. + + + + + + \ No newline at end of file diff --git a/planemo/xml/xsd/tool b/planemo/xml/xsd/tool new file mode 160000 index 000000000..ae723a187 --- /dev/null +++ b/planemo/xml/xsd/tool @@ -0,0 +1 @@ +Subproject commit ae723a1873bd6a249ba60b735dbbbb02e8427dad diff --git a/planemo/xml/xsd/tool_dependencies.xsd b/planemo/xml/xsd/tool_dependencies.xsd new file mode 100644 index 000000000..5e1ae27ff --- /dev/null +++ b/planemo/xml/xsd/tool_dependencies.xsd @@ -0,0 +1,187 @@ + + + + + Tool Shed tool_dependencies.xml Schema + Describes dependencies for a tool shed repository. + + + + + Collection of package definitions. + + + + + + + + + + + + A package is a type of tool dependency and the combination of the name and version provide a unique identifier for it. It can contain any number of <repository> tag. + + + + + + + + + Package name + + + + + Package version + + + + + + + + + + + + + + + + + + Package version + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ToolShed URL, leave empty in source control and ToolShd will populate this automatically. + + + + + Name of repository + + + + + Owner of repository + + + + + + Changeset revision + + + + + + Changeset revision + + + + + + + + + + + + + + + + + + + + + + Implementation version (must be 1.0) + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/planemo_ext/galaxy/tools/lint.py b/planemo_ext/galaxy/tools/lint.py index 948298f14..9fc065e98 100644 --- a/planemo_ext/galaxy/tools/lint.py +++ b/planemo_ext/galaxy/tools/lint.py @@ -7,21 +7,20 @@ LEVEL_ERROR = "error" -def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN): - import galaxy.tools.linters +def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN, extra_modules=[]): lint_context = LintContext(level=level) + lint_xml_with(lint_context, tool_xml, extra_modules) + return not lint_context.failed(fail_level) + + +def lint_xml_with(lint_context, tool_xml, extra_modules=[]): + import galaxy.tools.linters linter_modules = submodules.submodules(galaxy.tools.linters) + linter_modules.extend(extra_modules) for module in linter_modules: for (name, value) in inspect.getmembers(module): if callable(value) and name.startswith("lint_"): - lint_context.lint(module, name, value, tool_xml) - found_warns = lint_context.found_warns - found_errors = lint_context.found_errors - if fail_level == LEVEL_WARN: - lint_fail = (found_warns or found_errors) - else: - lint_fail = found_errors - return not lint_fail + lint_context.lint(name, value, tool_xml) class LintContext(object): @@ -31,14 +30,14 @@ def __init__(self, level): self.found_errors = False self.found_warns = False - def lint(self, module, name, lint_func, tool_xml): + def lint(self, name, lint_func, lint_target): name = name.replace("tsts", "tests") self.printed_linter_info = False self.valid_messages = [] self.info_messages = [] self.warn_messages = [] self.error_messages = [] - lint_func(tool_xml, self) + lint_func(lint_target, self) # TODO: colorful emoji if in click CLI. if self.error_messages: status = "FAIL" @@ -89,3 +88,12 @@ def error(self, message, *args): def warn(self, message, *args): self.__handle_message(self.warn_messages, message, *args) + + def failed(self, fail_level): + found_warns = self.found_warns + found_errors = self.found_errors + if fail_level == LEVEL_WARN: + lint_fail = (found_warns or found_errors) + else: + lint_fail = found_errors + return lint_fail diff --git a/planemo_ext/galaxy/tools/linters/xsd.py b/planemo_ext/galaxy/tools/linters/xsd.py new file mode 100644 index 000000000..a032a7f3a --- /dev/null +++ b/planemo_ext/galaxy/tools/linters/xsd.py @@ -0,0 +1,14 @@ +from ..lint import SkipLint + +try: + from lxml import etree +except ImportError: + etree = None + + +def lint_via_xsd(tool_xml, lint_ctx): + if not lint_ctx.use_schema: + raise SkipLint() + if etree is None: + lint_ctx.error("Requested linting via XML Schema but lxml is unavailable") + return diff --git a/planemo_ext/galaxy/tools/linters/xsd_util.py b/planemo_ext/galaxy/tools/linters/xsd_util.py new file mode 100644 index 000000000..f4086ccc0 --- /dev/null +++ b/planemo_ext/galaxy/tools/linters/xsd_util.py @@ -0,0 +1,58 @@ +import abc +from collections import namedtuple + +from galaxy.tools.deps.commands import which + +try: + from lxml import etree +except ImportError: + etree = None + + +class XsdValidator(object): + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def validate(self, schema_path, target_path): + """ Validate ``target_path`` against ``schema_path``. + + :return type: ValidationResult + """ + + @abc.abstractmethod + def enabled(self): + """ Return True iff system has dependencies for this validator. + + :return type: bool + """ + +ValidationResult = namedtuple("ValidationResult", ["passed", "output"]) + + +class LxmlValidator(XsdValidator): + """ Validate XSD files using lxml library. """ + + def validate(self, schema_path, target_path): + pass + + def enabled(self): + return etree is not None + + +class XmllintValidator(XsdValidator): + """ Validate XSD files with the external tool xmllint. """ + + def validate(self, schema_path, target_path): + pass + + def enabled(self): + return bool(which("xmllint")) + + +VALIDATORS = [LxmlValidator(), XmllintValidator()] + + +def get_validator(): + for validator in VALIDATORS: + if validator.enabled(): + return validator diff --git a/setup.py b/setup.py index 9a981b36f..0c3d509ef 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,9 @@ packages=[ 'planemo', 'planemo.commands', + 'planemo.linters', 'planemo.reports', + 'planemo.xml', 'planemo_ext', 'planemo_ext.galaxy', 'planemo_ext.galaxy.eggs', @@ -62,7 +64,14 @@ 'tool_factory_2/rgToolFactory2.xml', 'tool_factory_2/rgToolFactory2.py', 'tool_factory_2/getlocalrpackages.py', - ]}, + ], + 'planemo': ['xml/xsd/repository_dependencies.xsd', + 'xml/xsd/tool_dependencies.xsd', + 'xml/xsd/tool/galaxy.xsd', + 'xml/xsd/tool/citation.xsd', + 'xml/xsd/tool/citations.xsd', + ] + }, package_dir={'planemo': 'planemo', 'planemo_ext': 'planemo_ext'}, include_package_data=True, diff --git a/tests/repository_dependencies.xml b/tests/repository_dependencies.xml new file mode 100644 index 000000000..6fd02f07a --- /dev/null +++ b/tests/repository_dependencies.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/test_utils.py b/tests/test_utils.py index 568ef53c5..f9041f7a5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,6 +8,9 @@ from planemo import cli from sys import version_info +from galaxy.tools.deps.commands import which + + if version_info < (2, 7): from unittest2 import TestCase, skip else: @@ -58,6 +61,24 @@ def skip_unless_environ(var): return skip(template % var) +def skip_unless_module(module): + available = True + try: + __import__(module) + except ImportError: + available = False + if available: + return lambda func: func + template = "Module %s could not be loaded, dependent test skipped." + return skip(template % module) + + +def skip_unless_executable(executable): + if which(executable): + return lambda func: func + return skip("PATH doesn't contain executable %s" % executable) + + __all__ = [ "TestCase", "CliTestCase", diff --git a/tests/test_validation.py b/tests/test_validation.py new file mode 100644 index 000000000..4d54b0bff --- /dev/null +++ b/tests/test_validation.py @@ -0,0 +1,66 @@ +import os + +from .test_utils import ( + skip_unless_module, + skip_unless_executable, +) + +from planemo.xml import validation +from planemo import shed_lint + +TEST_PATH = os.path.dirname(__file__) + + +@skip_unless_module("lxml") +def test_lxml_validation(): + lxml_xsd_validator = validation.LxmlValidator() + _check_validator(lxml_xsd_validator) + + +@skip_unless_executable("xmllint") +def test_xmllint_validation(): + xmllint_xsd_validator = validation.XmllintValidator() + _check_validator(xmllint_xsd_validator) + + +def test_tool_dependencies_validation(): + _assert_validates(shed_lint.TOOL_DEPENDENCIES_XSD, + _path("tool_dependencies_good_1.xml")) + _assert_validates(shed_lint.TOOL_DEPENDENCIES_XSD, + _path("tool_dependencies_good_2.xml")) + + +def test_repository_dependencies_validation(): + _assert_validates(shed_lint.REPO_DEPENDENCIES_XSD, + _path("repository_dependencies.xml")) + + +def _check_validator(xsd_validator): + _assert_validates( + _path("xsd_schema_1.xsd"), + _path("xml_good_1.xml"), + xsd_validator, + ) + _assert_validates( + _path("xsd_schema_1.xsd"), + _path("xml_good_2.xml"), + xsd_validator, + ) + + result = xsd_validator.validate(_path("xsd_schema_1.xsd"), + _path("xml_bad_1.xml")) + assert not result.passed + output = result.output + assert "not_command" in str(output), str(output) + + +def _assert_validates(schema, target, xsd_validator=None): + if xsd_validator is None: + xsd_validator = validation.get_validator() + result = xsd_validator.validate(schema, + target) + assert result.passed, result.output + + +def _path(filename): + return os.path.join(TEST_PATH, filename) diff --git a/tests/tool_dependencies_good_1.xml b/tests/tool_dependencies_good_1.xml new file mode 100644 index 000000000..7dfe8f1e3 --- /dev/null +++ b/tests/tool_dependencies_good_1.xml @@ -0,0 +1,82 @@ + + + + + + + http://depot.galaxyproject.org/package/linux/i386/samtools/samtools-0.1.16-linux-i386.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/linux/x86_64/samtools/samtools-0.1.16-linux-x86_64.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/darwin/i386/samtools/samtools-0.1.16-Darwin-i386.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/darwin/x86_64/samtools/samtools-0.1.16-Darwin-x86_64.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/source/samtools/samtools-0.1.16.tar.bz2 + sed -i.bak 's/-lcurses/-lncurses/' Makefile + make + + samtools + $INSTALL_DIR/bin + + + libbam.a + $INSTALL_DIR/lib + + + + $INSTALL_DIR/bin + $INSTALL_DIR/lib + + + + + This is the last version of SAMTools to include the 'pileup' command. + + Program: samtools (Tools for alignments in the SAM format) + Version: 0.1.16 (r963:234) + + Usage: samtools <command> [options] + + Command: view SAM<->BAM conversion + sort sort alignment file + pileup generate pileup output + mpileup multi-way pileup + depth compute the depth + faidx index/extract FASTA + tview text alignment viewer + index index alignment + idxstats BAM index stats (r595 or later) + fixmate fix mate information + glfview print GLFv3 file + flagstat simple stats + calmd recalculate MD/NM tags and '=' bases + merge merge sorted alignments + rmdup remove PCR duplicates + reheader replace BAM header + cat concatenate BAMs + targetcut cut fosmid regions (for fosmid pool only) + phase phase heterozygotes + + + diff --git a/tests/tool_dependencies_good_2.xml b/tests/tool_dependencies_good_2.xml new file mode 100644 index 000000000..acee07390 --- /dev/null +++ b/tests/tool_dependencies_good_2.xml @@ -0,0 +1,102 @@ + + + + + + + http://depot.galaxyproject.org/package/darwin/x86_64/R/R-2.15.0-Darwin-x86_64.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/linux/x86_64/R/R-2.15.0-Linux-x86_64.tgz + + . + $INSTALL_DIR + + + $INSTALL_DIR/lib/libtcl8.4.so + $INSTALL_DIR/lib/libtk8.4.so + + + + + + + + + + + + + + + + + + + http://cran.rstudio.com/src/base/R-2/R-2.15.0.tar.gz + + + + + + + + + + + + + + + + + + + export LDFLAGS="-L$PNG_LIBS -L$READLINE_LIBS" && + export LDFLAGS="$LDFLAGS -Wl,-rpath,$READLINE_LIBS" && + export LDFLAGS="$LDFLAGS -Wl,-rpath,$PIXMAN_LIB_PATH" && + export LDFLAGS="$LDFLAGS -Wl,-rpath,$PNG_LIB_PATH" && + export LDFLAGS="$LDFLAGS -Wl,-rpath,$FREETYPE_LIB_PATH" && + export CFLAGS="-I$PNG_INCLUDES -I$READLINE_INCLUDES" && + export CXXFLAGS="-I$PNG_INCLUDES -I$READLINE_INCLUDES" && + export CPPFLAGS="-I$PNG_INCLUDES -I$READLINE_INCLUDES" && + ./configure --with-tcltk \ + --with-blas \ + --with-lapack \ + --with-readline \ + --with-cairo \ + --with-libpng \ + --without-x \ + --enable-R-shlib \ + --disable-R-framework \ + --libdir=$INSTALL_DIR/lib \ + --prefix=$INSTALL_DIR + + + sed -i.bak 's;$INSTALL_DIR;\${R_ROOT_DIR};g' $INSTALL_DIR/bin/R + sed -i.bak 's;$INSTALL_DIR;\${R_ROOT_DIR};g' $INSTALL_DIR/lib/R/bin/R + + + $INSTALL_DIR/lib/R/doc + $INSTALL_DIR/lib/R + $INSTALL_DIR/lib/R/include + $INSTALL_DIR/lib/R/share + $INSTALL_DIR + $INSTALL_DIR/lib/R + $INSTALL_DIR/lib/R + $INSTALL_DIR/lib/R/library + $INSTALL_DIR/bin + + + + + The precompiled versions of this package were built 2014-03-06. + R is a free software environment for statistical computing and graphics. + NOTE: See custom compilation options above. + + + \ No newline at end of file diff --git a/tests/xml_bad_1.xml b/tests/xml_bad_1.xml new file mode 100644 index 000000000..82d6fc40e --- /dev/null +++ b/tests/xml_bad_1.xml @@ -0,0 +1,2 @@ + + diff --git a/tests/xml_good_1.xml b/tests/xml_good_1.xml new file mode 100644 index 000000000..f3d3141c9 --- /dev/null +++ b/tests/xml_good_1.xml @@ -0,0 +1,2 @@ + + diff --git a/tests/xml_good_2.xml b/tests/xml_good_2.xml new file mode 100644 index 000000000..94cd80141 --- /dev/null +++ b/tests/xml_good_2.xml @@ -0,0 +1,3 @@ + + moo.py --arg 1 + diff --git a/tests/xsd_command.xsd b/tests/xsd_command.xsd new file mode 100644 index 000000000..283601042 --- /dev/null +++ b/tests/xsd_command.xsd @@ -0,0 +1,23 @@ + + + + + + This tag specifies how Galaxy should invoke the tool's executable, passing its required input parameter values (the command line specification links the parameters supplied in the form with the actual tool executable). Any word inside it starting with a dollar sign ($) will be treated as a variable whose values can be acquired from one of three sources: parameters, metadata, or output files. After the substitution of variables with their values, the content is interpreted with Cheetah and finally given to the interpreter specified in the corresponding attribute (if any). + + + + + + This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. + + + + + + + \ No newline at end of file diff --git a/tests/xsd_schema_1.xsd b/tests/xsd_schema_1.xsd new file mode 100644 index 000000000..24696e8d8 --- /dev/null +++ b/tests/xsd_schema_1.xsd @@ -0,0 +1,9 @@ + + + + + \ No newline at end of file