From 912df026256150bbb3005c1b8462075d26de1062 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 20 Feb 2015 16:08:50 -0500 Subject: [PATCH 1/2] Use XSD to lint tools and repositories. - Implement validation abstraction that will use either lxml (Python lib) or xmllint (command line app) dependending on what is available - with test cases. - Add a ``--xsd`` flag to the lint command that lints against the experimental XSD from https://github.com/JeanFred/Galaxy-XSD. - Implement ``repository_dependencies.xsd`` to describe Tool Shed ``repository_dependencies.xml`` files (fairly complete). - Implement ``tool_dependencies.xsd`` to describe Tool Shed ``tool_dependencies.xml`` files. - Validates attributes and elements down to the ``action`` elements and then largely gives up (sticking ``any`` and ``anyAttribute`` tags on that element). - Registers everything in tools-devteam as valid, and detects one invalid XML file in tools-iuc. - Implement new ``shed_lint`` command that: - Validates tool_dependencies.xml against schema - Validates repository_dependencies.xml against schema - Bare minimum to lint .shed.yml files. - Optionally also lints tools in repsitories with the ``--tools`` argument. - Can recursively lint many repositories at ont time ``-r``. - Refactoring of existing stuff to support this and make room for XSD validation of tool XML files and generalizing applying actions over many repositories and many tools. --- planemo/commands/cmd_lint.py | 49 +- planemo/commands/cmd_shed_lint.py | 34 + planemo/commands/cmd_shed_upload.py | 25 +- planemo/galaxy_test.py | 1 + planemo/lint.py | 15 + planemo/linters/__init__.py | 0 planemo/linters/xsd.py | 30 + planemo/options.py | 35 + planemo/shed.py | 24 +- planemo/shed_lint.py | 87 ++ planemo/tool_lint.py | 72 + planemo/xml/__init__.py | 6 + planemo/xml/validation.py | 81 ++ planemo/xml/xsd/repository_dependencies.xsd | 60 + planemo/xml/xsd/tool/LICENSE | 22 + planemo/xml/xsd/tool/citation.xsd | 30 + planemo/xml/xsd/tool/citations.xsd | 15 + planemo/xml/xsd/tool/galaxy.xsd | 1277 ++++++++++++++++++ planemo/xml/xsd/tool_dependencies.xsd | 187 +++ planemo_ext/galaxy/tools/lint.py | 32 +- planemo_ext/galaxy/tools/linters/xsd.py | 14 + planemo_ext/galaxy/tools/linters/xsd_util.py | 58 + scripts/update_extern.sh | 7 + setup.py | 11 +- tests/repository_dependencies.xml | 18 + tests/test_utils.py | 21 + tests/test_validation.py | 66 + tests/tool_dependencies_good_1.xml | 82 ++ tests/tool_dependencies_good_2.xml | 102 ++ tests/xml_bad_1.xml | 2 + tests/xml_good_1.xml | 2 + tests/xml_good_2.xml | 3 + tests/xsd_command.xsd | 23 + tests/xsd_schema_1.xsd | 9 + 34 files changed, 2428 insertions(+), 72 deletions(-) create mode 100644 planemo/commands/cmd_shed_lint.py create mode 100644 planemo/lint.py create mode 100644 planemo/linters/__init__.py create mode 100644 planemo/linters/xsd.py create mode 100644 planemo/shed_lint.py create mode 100644 planemo/tool_lint.py create mode 100644 planemo/xml/__init__.py create mode 100644 planemo/xml/validation.py create mode 100644 planemo/xml/xsd/repository_dependencies.xsd create mode 100644 planemo/xml/xsd/tool/LICENSE create mode 100644 planemo/xml/xsd/tool/citation.xsd create mode 100644 planemo/xml/xsd/tool/citations.xsd create mode 100644 planemo/xml/xsd/tool/galaxy.xsd create mode 100644 planemo/xml/xsd/tool_dependencies.xsd create mode 100644 planemo_ext/galaxy/tools/linters/xsd.py create mode 100644 planemo_ext/galaxy/tools/linters/xsd_util.py create mode 100644 tests/repository_dependencies.xml create mode 100644 tests/test_validation.py create mode 100644 tests/tool_dependencies_good_1.xml create mode 100644 tests/tool_dependencies_good_2.xml create mode 100644 tests/xml_bad_1.xml create mode 100644 tests/xml_good_1.xml create mode 100644 tests/xml_good_2.xml create mode 100644 tests/xsd_command.xsd create mode 100644 tests/xsd_schema_1.xsd diff --git a/planemo/commands/cmd_lint.py b/planemo/commands/cmd_lint.py index cae1d8378..769b9b9f6 100644 --- a/planemo/commands/cmd_lint.py +++ b/planemo/commands/cmd_lint.py @@ -1,55 +1,24 @@ import sys -import traceback + import click from planemo.cli import pass_context -from planemo.io import info -from planemo.io import error from planemo import options -from galaxy.tools.loader_directory import load_tool_elements_from_path -from galaxy.tools.lint import lint_xml - -SKIP_XML_MESSAGE = "Skipping XML file - does not appear to be a tool %s." -LINTING_TOOL_MESSAGE = "Linting tool %s" +from planemo.tool_lint import build_lint_args +from planemo.tool_lint import lint_tools_on_path @click.command('lint') @options.optional_tools_arg() -@click.option( - '--report_level', - type=click.Choice(['all', 'warn', 'error']), - default="all" -) -@click.option( - '--fail_level', - type=click.Choice(['warn', 'error']), - default="warn" -) +@options.report_level_option() +@options.fail_level_option() +@options.lint_xsd_option() @pass_context -def cli(ctx, path, report_level="all", fail_level="warn"): +def cli(ctx, path, **kwds): """Check specified tool(s) for common errors and adherence to best practices. """ - exit = 0 - lint_args = dict(level=report_level, fail_level=fail_level) - tools = load_tool_elements_from_path(path, load_exception_handler) - valid_tools = 0 - for (tool_path, tool_xml) in tools: - if tool_xml.getroot().tag != "tool": - if ctx.verbose: - info(SKIP_XML_MESSAGE % tool_path) - continue - info("Linting tool %s" % tool_path) - if not lint_xml(tool_xml, **lint_args): - exit = 1 - else: - valid_tools += 1 - if exit == 0 and valid_tools == 0: - exit = 2 + lint_args = build_lint_args(**kwds) + exit = lint_tools_on_path(ctx, path, lint_args) sys.exit(exit) - - -def load_exception_handler(path, exc_info): - error("Error loading tool with path %s" % path) - traceback.print_exception(*exc_info, limit=1, file=sys.stderr) diff --git a/planemo/commands/cmd_shed_lint.py b/planemo/commands/cmd_shed_lint.py new file mode 100644 index 000000000..aed5b2e45 --- /dev/null +++ b/planemo/commands/cmd_shed_lint.py @@ -0,0 +1,34 @@ +import click +import sys + +from planemo.cli import pass_context +from planemo import options +from planemo import shed +from planemo import shed_lint + + +@click.command('shed_lint') +@options.optional_project_arg(exists=True) +@options.report_level_option() +@options.fail_level_option() +@options.click.option( + '--tools', + is_flag=True, + default=False, + help=("Lint tools discovered in the process of linting repositories.") +) +@options.lint_xsd_option() +@options.recursive_shed_option() +@pass_context +def cli(ctx, path, recursive=False, **kwds): + """Check a Tool Shed repository for common problems. + """ + def lint(path): + return shed_lint.lint_repository(ctx, path, **kwds) + + if recursive: + exit_code = shed.for_each_repository(lint, path) + else: + exit_code = lint(path) + + sys.exit(exit_code) diff --git a/planemo/commands/cmd_shed_upload.py b/planemo/commands/cmd_shed_upload.py index 375214e12..da69d29f7 100644 --- a/planemo/commands/cmd_shed_upload.py +++ b/planemo/commands/cmd_shed_upload.py @@ -10,8 +10,6 @@ from planemo.io import shell import json -import fnmatch -import os tar_path = click.Path( exists=True, @@ -54,15 +52,15 @@ is_flag=True, default=False ) -@click.option( - '-r', '--recursive', - is_flag=True, - help="Recursively search for repositories to publish to a tool shed", -) +@options.recursive_shed_option() @pass_context def cli(ctx, path, **kwds): """Handle possible recursion through paths for uploading files to a toolshed """ + + def upload(path): + return __handle_upload(ctx, **kwds) + if kwds['recursive']: if kwds['name'] is not None: error("--name is incompatible with --recursive") @@ -71,17 +69,9 @@ def cli(ctx, path, **kwds): error("--tar is incompatible with --recursive") return -1 - ret_codes = [] - for base_path, dirnames, filenames in os.walk(path): - for filename in fnmatch.filter(filenames, '.shed.yml'): - ret_codes.append( - __handle_upload(ctx, base_path, **kwds) - ) - # "Good" returns are Nones, everything else is a -1 and should be - # passed upwards. - return None if all(x is None for x in ret_codes) else -1 + return shed.for_each_repository(upload, path) else: - return __handle_upload(ctx, path, **kwds) + return upload(path) def __handle_upload(ctx, path, **kwds): @@ -121,6 +111,7 @@ def __handle_upload(ctx, path, **kwds): error(e2.read()) return -1 info("Repository %s updated successfully." % path) + return 0 def __find_repository(ctx, tsi, path, **kwds): diff --git a/planemo/galaxy_test.py b/planemo/galaxy_test.py index 8655ece54..0f2bc045b 100644 --- a/planemo/galaxy_test.py +++ b/planemo/galaxy_test.py @@ -1,5 +1,6 @@ """ Utilities for reasoning about Galaxy test results. """ +from __future__ import absolute_import import json import xml.etree.ElementTree as ET diff --git a/planemo/lint.py b/planemo/lint.py new file mode 100644 index 000000000..d36760201 --- /dev/null +++ b/planemo/lint.py @@ -0,0 +1,15 @@ +import os + +from planemo.xml import validation + + +def lint_xsd(lint_ctx, schema_path, path): + name = os.path.basename(path) + validator = validation.get_validator(require=True) + validation_result = validator.validate(schema_path, path) + if not validation_result.passed: + msg = "Invalid %s found. Errors [%s]" + msg = msg % (name, validation_result.output) + lint_ctx.error(msg) + else: + lint_ctx.info("%s found and appears to be valid XML" % name) diff --git a/planemo/linters/__init__.py b/planemo/linters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/planemo/linters/xsd.py b/planemo/linters/xsd.py new file mode 100644 index 000000000..4b1689e93 --- /dev/null +++ b/planemo/linters/xsd.py @@ -0,0 +1,30 @@ +""" Tool linting module that lints Galaxy tool against experimental XSD. +""" +import copy +import os +import tempfile + +from planemo.xml import XSDS_PATH +import planemo.lint + +TOOL_XSD = os.path.join(XSDS_PATH, "tool", "galaxy.xsd") + + +def lint_tool_xsd(root, lint_ctx): + """ Write a temp file out and lint it. + """ + with tempfile.NamedTemporaryFile() as tf: + _clean_root(root).write(tf.name) + planemo.lint.lint_xsd(lint_ctx, TOOL_XSD, tf.name) + + +def _clean_root(root): + """ XSD assumes macros have been expanded, so remove them. + """ + clean_root = copy.deepcopy(root) + to_remove = [] + for macros_el in clean_root.findall("macros"): + to_remove.append(macros_el) + for macros_el in to_remove: + clean_root.getroot().remove(macros_el) + return clean_root diff --git a/planemo/options.py b/planemo/options.py index dca77e5f0..671ff8a65 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -249,3 +249,38 @@ def shed_password_option(): help="Password for Tool Shed auth (required unless shed_key is " "specified)." ) + + +def lint_xsd_option(): + return click.option( + '--xsd', + is_flag=True, + default=False, + help=("Include experimental tool XSD validation in linting " + "process (requires xmllint on PATH or lxml installed).") + ) + + +def report_level_option(): + return click.option( + '--report_level', + type=click.Choice(['all', 'warn', 'error']), + default="all", + ) + + +def fail_level_option(): + return click.option( + '--fail_level', + type=click.Choice(['warn', 'error']), + default="warn" + ) + + +def recursive_shed_option(): + return click.option( + '-r', + '--recursive', + is_flag=True, + help="Recursively perform command for nested repository directories.", + ) diff --git a/planemo/shed.py b/planemo/shed.py index f1865a9d8..6c8042b62 100644 --- a/planemo/shed.py +++ b/planemo/shed.py @@ -1,8 +1,10 @@ +import fnmatch +import glob import os -from tempfile import mkstemp import tarfile +from tempfile import mkstemp + import yaml -import glob try: from bioblend import toolshed @@ -192,6 +194,24 @@ def build_tarball(tool_path): return temp_path +def walk_repositories(path): + """ Recurse through directories and find effective repositories. """ + for base_path, dirnames, filenames in os.walk(path): + for filename in fnmatch.filter(filenames, '.shed.yml'): + yield base_path + + +def for_each_repository(function, path): + ret_codes = [] + for base_path in walk_repositories(path): + ret_codes.append( + function(base_path) + ) + # "Good" returns are Nones, everything else is a -1 and should be + # passed upwards. + return 0 if all((not x) for x in ret_codes) else -1 + + def username(tsi): user = _user(tsi) return user["username"] diff --git a/planemo/shed_lint.py b/planemo/shed_lint.py new file mode 100644 index 000000000..a274e0a36 --- /dev/null +++ b/planemo/shed_lint.py @@ -0,0 +1,87 @@ +import os +import yaml +from galaxy.tools.lint import LintContext +from planemo.lint import lint_xsd +from planemo.tool_lint import ( + build_lint_args, + yield_tool_xmls, +) +from planemo.xml import XSDS_PATH + + +from planemo.io import info +from planemo.io import error + +from galaxy.tools.lint import lint_xml_with + +TOOL_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "tool_dependencies.xsd") +REPO_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "repository_dependencies.xsd") + + +def lint_repository(ctx, path, **kwds): + info("Linting repository %s" % path) + lint_args = build_lint_args(**kwds) + lint_ctx = LintContext(lint_args["level"]) + lint_ctx.lint( + "tool_dependencies", + lint_tool_dependencies, + path, + ) + lint_ctx.lint( + "repository_dependencies", + lint_repository_dependencies, + path, + ) + lint_ctx.lint( + "shed_yaml", + lint_shed_yaml, + path, + ) + if kwds["tools"]: + for (tool_path, tool_xml) in yield_tool_xmls(ctx, path): + info("+Linting tool %s" % tool_path) + lint_xml_with( + lint_ctx, + tool_xml, + extra_modules=lint_args["extra_modules"] + ) + failed = lint_ctx.failed(lint_args["fail_level"]) + if failed: + error("Failed linting") + return 1 if failed else 0 + + +def lint_tool_dependencies(path, lint_ctx): + tool_dependencies = os.path.join(path, "tool_dependencies.xml") + if not os.path.exists(tool_dependencies): + lint_ctx.info("No tool_dependencies.xml, skipping.") + return + lint_xsd(lint_ctx, TOOL_DEPENDENCIES_XSD, tool_dependencies) + + +def lint_repository_dependencies(path, lint_ctx): + repo_dependencies = os.path.join(path, "repository_dependencies.xml") + if not os.path.exists(repo_dependencies): + lint_ctx.info("No repository_dependencies.xml, skipping.") + return + lint_xsd(lint_ctx, REPO_DEPENDENCIES_XSD, repo_dependencies) + + +def lint_shed_yaml(path, lint_ctx): + shed_yaml = os.path.join(path, ".shed.yml") + if not os.path.exists(shed_yaml): + lint_ctx.info("No .shed.yml file found, skipping.") + return + try: + shed_contents = yaml.load(open(shed_yaml, "r")) + except Exception as e: + lint_ctx.warn("Failed to parse .shed.yml file [%s]" % str(e)) + + warned = False + for required_key in ["owner", "name"]: + if required_key not in shed_contents: + lint_ctx.warn(".shed.yml did not contain key [%s]" % required_key) + warned = True + + if not warned: + lint_ctx.info(".shed.yml found and appears to be valid YAML.") diff --git a/planemo/tool_lint.py b/planemo/tool_lint.py new file mode 100644 index 000000000..fca04d03e --- /dev/null +++ b/planemo/tool_lint.py @@ -0,0 +1,72 @@ +import os +import sys +import traceback + +from planemo.io import info +from planemo.io import error + +import planemo.linters.xsd + +from galaxy.tools.loader_directory import load_tool_elements_from_path +from galaxy.tools.lint import lint_xml + +SKIP_XML_MESSAGE = "Skipping XML file - does not appear to be a tool %s." +LINTING_TOOL_MESSAGE = "Linting tool %s" +SHED_FILES = ["tool_dependencies.xml", "repository_dependencies.xml"] + + +def lint_tools_on_path(ctx, path, lint_args, assert_tools=True): + exit = 0 + valid_tools = 0 + for (tool_path, tool_xml) in yield_tool_xmls(ctx, path): + info("Linting tool %s" % tool_path) + if not lint_xml(tool_xml, **lint_args): + error("Failed linting") + exit = 1 + else: + valid_tools += 1 + if exit == 0 and valid_tools == 0 and assert_tools: + exit = 2 + return exit + + +def yield_tool_xmls(ctx, path): + tools = load_tool_elements_from_path(path, load_exception_handler) + for (tool_path, tool_xml) in tools: + if not _is_tool_xml(ctx, tool_path, tool_xml): + continue + yield (tool_path, tool_xml) + + +def build_lint_args(**kwds): + report_level = kwds.get("report_level", "all") + fail_level = kwds.get("fail_level", "warn") + lint_args = dict( + level=report_level, + fail_level=fail_level, + extra_modules=_lint_extra_modules(**kwds) + ) + return lint_args + + +def load_exception_handler(path, exc_info): + error("Error loading tool with path %s" % path) + traceback.print_exception(*exc_info, limit=1, file=sys.stderr) + + +def _lint_extra_modules(**kwds): + xsd = kwds.get("xsd", False) + if xsd: + return [planemo.linters.xsd] + else: + return [] + + +def _is_tool_xml(ctx, tool_path, tool_xml): + if os.path.basename(tool_path) in SHED_FILES: + return False + if tool_xml.getroot().tag != "tool": + if ctx.verbose: + info(SKIP_XML_MESSAGE % tool_path) + return False + return True diff --git a/planemo/xml/__init__.py b/planemo/xml/__init__.py new file mode 100644 index 000000000..585f56cb1 --- /dev/null +++ b/planemo/xml/__init__.py @@ -0,0 +1,6 @@ +import os + +XML_DIRECTORY = os.path.dirname(__file__) +XSDS_PATH = os.path.join(XML_DIRECTORY, "xsd") + +__all__ = [XSDS_PATH] diff --git a/planemo/xml/validation.py b/planemo/xml/validation.py new file mode 100644 index 000000000..311f30d1f --- /dev/null +++ b/planemo/xml/validation.py @@ -0,0 +1,81 @@ +import abc +from collections import namedtuple +import subprocess + +from galaxy.tools.deps.commands import which + +try: + from lxml import etree +except ImportError: + etree = None + +XMLLINT_COMMAND = "xmllint --noout --schema {0} {1} 2>&1" +INSTALL_VALIDATOR_MESSAGE = ("This feature requires an external dependency " + "to function, pleaes install xmllint (e.g 'brew " + "install libxml2' or 'apt-get install " + "libxml2-utils'.") + + +class XsdValidator(object): + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def validate(self, schema_path, target_path): + """ Validate ``target_path`` against ``schema_path``. + + :return type: ValidationResult + """ + + @abc.abstractmethod + def enabled(self): + """ Return True iff system has dependencies for this validator. + + :return type: bool + """ + +ValidationResult = namedtuple("ValidationResult", ["passed", "output"]) + + +class LxmlValidator(XsdValidator): + """ Validate XSD files using lxml library. """ + + def validate(self, schema_path, target_path): + try: + xsd_doc = etree.parse(schema_path) + xsd = etree.XMLSchema(xsd_doc) + xml = etree.parse(target_path) + passed = xsd.validate(xml) + return ValidationResult(passed, xsd.error_log) + except etree.XMLSyntaxError as e: + return ValidationResult(False, str(e)) + + def enabled(self): + return etree is not None + + +class XmllintValidator(XsdValidator): + """ Validate XSD files with the external tool xmllint. """ + + def validate(self, schema_path, target_path): + command = XMLLINT_COMMAND.format(schema_path, target_path) + p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + stdout, _ = p.communicate() + passed = p.returncode == 0 + return ValidationResult(passed, stdout) + + def enabled(self): + return bool(which("xmllint")) + + +VALIDATORS = [LxmlValidator(), XmllintValidator()] + + +def get_validator(require=True): + for validator in VALIDATORS: + if validator.enabled(): + return validator + + if require: + raise Exception(INSTALL_VALIDATOR_MESSAGE) + + return None diff --git a/planemo/xml/xsd/repository_dependencies.xsd b/planemo/xml/xsd/repository_dependencies.xsd new file mode 100644 index 000000000..4c4c6e5ff --- /dev/null +++ b/planemo/xml/xsd/repository_dependencies.xsd @@ -0,0 +1,60 @@ + + + + + Tool Shed repository_dependencies.xml Schema + Describes dependencies this repository depends on for things other then tool dependencies. + + + + + Collection of repositories. + + + + + + + Description of repositries. + + + + + + + Defines a specific revision of a repository on which this repository depends. If the toolshed is not defined, it will be automatically set to the local Tool Shed. If defined, the changeset_revision is the minimum required version. If the changeset_revision is not defined, it will be set to the latest installable changeset_revision for the repository defined by the name and owner. If either the toolshed or the changeset_revision is not defined, the repository_dependencies.xml file will automatically be altered (before it is committed in the changeset) to include the attributes and values just discussed. + + + + Repository name + + + + + Owner of repository + + + + + Tool Shed containing repository. + + + + + Changeset revision of dependent repository. + + + + + Prior installation required. + + + + + + \ No newline at end of file diff --git a/planemo/xml/xsd/tool/LICENSE b/planemo/xml/xsd/tool/LICENSE new file mode 100644 index 000000000..bfa63c52c --- /dev/null +++ b/planemo/xml/xsd/tool/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2011-2015 John Chilton, Jean-Frédéric Berthelot, Pierre Lindenbaum + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/planemo/xml/xsd/tool/citation.xsd b/planemo/xml/xsd/tool/citation.xsd new file mode 100644 index 000000000..81c470687 --- /dev/null +++ b/planemo/xml/xsd/tool/citation.xsd @@ -0,0 +1,30 @@ + + + + + + + Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. + + + + + + type of citation + + + + + + + + + Type of citation represented. + + + + + + + + diff --git a/planemo/xml/xsd/tool/citations.xsd b/planemo/xml/xsd/tool/citations.xsd new file mode 100644 index 000000000..3db22e7c9 --- /dev/null +++ b/planemo/xml/xsd/tool/citations.xsd @@ -0,0 +1,15 @@ + + + + + + + Tool files may declare one citations element. Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. + + + + + + + + diff --git a/planemo/xml/xsd/tool/galaxy.xsd b/planemo/xml/xsd/tool/galaxy.xsd new file mode 100644 index 000000000..fbcde8349 --- /dev/null +++ b/planemo/xml/xsd/tool/galaxy.xsd @@ -0,0 +1,1277 @@ + + + + Galaxy Schema + A Galaxy XML tool wrapper + + + + + + + + The outer-most tag set + + + + + + + The value is displayed in the tool menu immediately following the hyperlink for the tool + + + + + + Specifies the command to be run in order to get the tool's version string. The resulting value will be found in the "Info" field of the history dataset. + + + + + + + + + + + Documentation for help + + + + + + + + + Must be unique across all tools; should be lowercase and contain only letters, numbers, and underscores. It allows for tool versioning and metrics of the number of times a tool is used, among other things. + + + + + This string is what is displayed as a hyperlink in the tool menu + + + + + This string defaults to "1.0.0' if it is not included in the tag. It allows for tool versioning and should be changed with each new version of the tool. + + + + + Allows for tools to be loaded upon server startup, but not displayed in the tool menu + + + + + Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. + + + + + Only if "tool_type" attribute value is "data_source" - defines the HTTP request method to use when communicating with an external data source application ( the default is "get" ). + + + + + + + + This is a container tag set for the 'requirement' tag set described below. + + + + + + + + + + This tag set is contained within the 'requirements' tag set. Third party programs or modules that the tool depends upon (and which are not distributed with Galaxy) are included in this tag set. The intention is that when Galaxy starts it can check whether the required programs or modules are available, and if not this tool will not be loaded. + + + + + + This value defines the which type of the 3rd party module required by this tool + + + + + Required for package type requirements + + + + + + + + This tag set is contained within the 'requirements' tag set. Galaxy can be configured to run tools within Docker (https://www.docker.com/) containers - this tag allows the tool to suggest possible valid Docker containers for this tool. + + + + + + + This value describes the type of container that the tool may be executed in and currently must be 'docker'. + + + + + + + + + Documentation for Parallelism + + + + Documentation for method + + + + + + Documentation for Code + + + + Documentation for file + + + + + + Documentation for UIhints + + + + Documentation for minwidth + + + + + + Documentation for Tests + + + + + + + + Documentation for Test + + + + + + + + + Documentation for TestParam + + + + + + + Documentation for name + + + + + Documentation for value + + + + + Documentation for ftype + + + + + Documentation for dbkey + + + + + + Definition of a collection for test input. + + + + + + + Type of collection to create. + + + + + + Documentation for TestOutput + + + + + + + Documentation for name + + + + + Documentation for file + + + + + Documentation for ftype + + + + + Documentation for sort + + + + + + Documentation for TestAssertion + + + + + + + + + + + + + + + Line the test output must contain. + + + + + + Consists of all tag sets that define the tool's input parameters. Each <param> tag within the <inputs> tag set maps to a command line parameter within the 'command' tag set described above. + + + + + + + Documentation for action + + + + + Documentation for check_values + + + + + Documentation for method + + + + + Documentation for target + + + + + + + + + + + + + Documentation for display + + + + + + + Documentation for InputType + + + + + Name for this element + + + + + + + + This is a container for conditional parameters in the tool ( must contain 'when' tag sets ) - the command line is wrapped in an if-else statement. + + + + + + + + + + + + + + + + Documentation for ConditionalWhen + + + + + + + Documentation for value + + + + + + + + + This is a container for any tag sets that can be contained within the 'inputs' tag set. When this is used, the tool will allow the user to add any number of additional sets of the contained parameters ( an "Add new 'title'" button will be displayed on the tool form ). + + + + + + + + + The title of the repeat section, which will be displayed on the tool form + + + + + The minimum number of repeat units + + + + + The maximum number of repeat units + + + + + The default number of repeat units + + + + + + + + + + + Documentation for Param + + + + + + + Documentation for label + + + + + + + + + + + + Documentation for type + + + + + Documentation for label + + + + + Documentation for help + + + + + Documentation for value + + + + + Documentation for optional + + + + + Documentation for min + + + + + Documentation for max + + + + + Documentation for format + + + + + Restrict valid collection types for data_collection params (e.g. list, paired, list:paired, etc...). + + + + + Documentation for data_ref + + + + + Documentation for accept_default + + + + + Documentation for force_select + + + + + Documentation for display + + + + + Documentation for multiple + + + + + Documentation for numerical + + + + + Documentation for hierarchy + + + + + Documentation for checked + + + + + Documentation for truevalue + + + + + Documentation for falsevalue + + + + + Documentation for size + + + + + Documentation for metadata_name + + + + + + + + Documentation for ParamType + + + + + + + + + + + + + + + + + + + + + + This tag specifies how Galaxy should invoke the tool's executable, passing its required input parameter values (the command line specification links the parameters supplied in the form with the actual tool executable). Any word inside it starting with a dollar sign ($) will be treated as a variable whose values can be acquired from one of three sources: parameters, metadata, or output files. After the substitution of variables with their values, the content is interpreted with Cheetah and finally given to the interpreter specified in the corresponding attribute (if any). + + + + + + This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. + + + + + + + + + + Documentation for ParamOption + + + + + + Documentation for value + + + + + Documentation for selected + + + + + + + + + + Documentation for Options + + + + + + + Documentation for from_dataset + + + + + Documentation for from_file + + + + + Documentation for from_data_table + + + + + Documentation for from_parameter + + + + + Documentation for options_filter_attribute + + + + + Documentation for transform_lines + + + + + Documentation for startswith + + + + + + + + + + + Documentation for file + + + + + + + Documentation for Column + + + + Documentation for name + + + + + Documentation for index + + + + + + Documentation for Validator + + + + Documentation for type + + + + + Documentation for message + + + + + Documentation for filename + + + + + Documentation for metadata_name + + + + + Documentation for metadata_column + + + + + Documentation for line_startswith + + + + + Documentation for min + + + + + Documentation for max + + + + + Documentation for split + + + + + + Documentation for Sanitizer + + + + + + + Documentation for sanitize + + + + + Documentation for invalid_char + + + + + + Documentation for Valid + + + + + + + Documentation for initial + + + + + + Documentation for ValidElementType + + + + Documentation for preset + + + + + Documentation for value + + + + + + Documentation for Add + + + + + + + + Documentation for Remove + + + + + + + + + + + + + + Documentation for Filter + + + + Documentation for type + + + + + Documentation for name + + + + + Documentation for column + + + + + Documentation for ref + + + + + Documentation for key + + + + + Documentation for separator + + + + + Documentation for compare + + + + + Documentation for keep + + + + + Documentation for value + + + + + Documentation for ref_attribute + + + + + + Documentation for Outputs + + + + + + + + Documentation for Data + + + + + + Documentation for filter + + + + + + + Documentation for name + + + + + Documentation for format + + + + + Documentation for format_source + + + + + Documentation for metadata_source + + + + + Documentation for label + + + + + Documentation for from_work_dir + + + + + Documentation for hidden + + + + + + Documentation for Actions + + + + + + + + + + + + + + Documentation for Action + + + + + + + Documentation for type + + + + + Documentation for name + + + + + Documentation for default + + + + + + Documentation for ActionsOption + + + + + + + Documentation for type + + + + + Documentation for name + + + + + Documentation for column + + + + + Documentation for offset + + + + + Documentation for param_attribute + + + + + + Documentation for ActionsConditional + + + + + + + + + + + + Documentation for ActionsConditionalWhen + + + + + + + Documentation for value + + + + + + Documentation for ConfigFiles + + + + + + + + Documentation for ConfigFile + + + + + + Documentation for name + + + + + Documentation for type + + + + + + + + + + Galaxy currently supports using regular expressions to scan stdout and stderr, and it also allows exit codes to be scanned for ranges. The 'stdio' tag has two subtags, 'regex' and 'exit_code', to define regular expressions and exit code processing, respectively. + + + + + + + + + + + Documentation for ExitCode + + + + Documentation for range + + + + + Documentation for level + + + + + Documentation for description + + + + + + + + A regular expression defines a pattern of characters + + + + This tells whether the regular expression should be matched against stdout, stderr, or both. If this attribute is missing or is incorrect, then both stdout and stderr will be checked. The source can be one of the follwing values: + + + + + This is the regular expression that will be used to match against stdout and/or stderr. + + + + + This works very similarly to the 'exit_code' tag, except that, when a regular expression matches against its source, the description is added to the beginning of the source. + + + + + an optional description of the regular expression that has matched. + + + + + + + + Documentation for ConfigFileType + + + + + + + + + + Documentation for ChangeFormat + + + + + + + + Documentation for WhenData + + + + + Documentation for input + + + + + Documentation for value + + + + + Documentation for format + + + + + + Documentation for RequirementType + + + + + + + + + + Type of container for tool execution. + + + + + + + + Documentation for ToolTypeType + + + + + + + + Documentation for URLmethodType + + + + + + + + + Documentation for TargetType + + + + + + + + + Documentation for MethodType + + + + + + + + + Documentation for DisplayType + + + + + + + + + Documentation for HierarchyType + + + + + + + + + Documentation for CheckedType + + + + + + + + + + + + + Documentation for ValidatorType + + + + + + + + + + + + + + + + + Documentation for FilterType + + + + + + + + + + + + + + Documentation for ActionType + + + + + + + + + Documentation for ActionsOptionType + + + + + + + + + + Documentation for CompareType + + + + + + + + Documentation for LevelType + + + + + + + + + + Documentation for RangeType + + + + + + + + Documentation for SourceType + + + + + + + + + + Documentation for PermissiveBoolean + + + + + + + + + + + diff --git a/planemo/xml/xsd/tool_dependencies.xsd b/planemo/xml/xsd/tool_dependencies.xsd new file mode 100644 index 000000000..5e1ae27ff --- /dev/null +++ b/planemo/xml/xsd/tool_dependencies.xsd @@ -0,0 +1,187 @@ + + + + + Tool Shed tool_dependencies.xml Schema + Describes dependencies for a tool shed repository. + + + + + Collection of package definitions. + + + + + + + + + + + + A package is a type of tool dependency and the combination of the name and version provide a unique identifier for it. It can contain any number of <repository> tag. + + + + + + + + + Package name + + + + + Package version + + + + + + + + + + + + + + + + + + Package version + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ToolShed URL, leave empty in source control and ToolShd will populate this automatically. + + + + + Name of repository + + + + + Owner of repository + + + + + + Changeset revision + + + + + + Changeset revision + + + + + + + + + + + + + + + + + + + + + + Implementation version (must be 1.0) + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/planemo_ext/galaxy/tools/lint.py b/planemo_ext/galaxy/tools/lint.py index 948298f14..9fc065e98 100644 --- a/planemo_ext/galaxy/tools/lint.py +++ b/planemo_ext/galaxy/tools/lint.py @@ -7,21 +7,20 @@ LEVEL_ERROR = "error" -def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN): - import galaxy.tools.linters +def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN, extra_modules=[]): lint_context = LintContext(level=level) + lint_xml_with(lint_context, tool_xml, extra_modules) + return not lint_context.failed(fail_level) + + +def lint_xml_with(lint_context, tool_xml, extra_modules=[]): + import galaxy.tools.linters linter_modules = submodules.submodules(galaxy.tools.linters) + linter_modules.extend(extra_modules) for module in linter_modules: for (name, value) in inspect.getmembers(module): if callable(value) and name.startswith("lint_"): - lint_context.lint(module, name, value, tool_xml) - found_warns = lint_context.found_warns - found_errors = lint_context.found_errors - if fail_level == LEVEL_WARN: - lint_fail = (found_warns or found_errors) - else: - lint_fail = found_errors - return not lint_fail + lint_context.lint(name, value, tool_xml) class LintContext(object): @@ -31,14 +30,14 @@ def __init__(self, level): self.found_errors = False self.found_warns = False - def lint(self, module, name, lint_func, tool_xml): + def lint(self, name, lint_func, lint_target): name = name.replace("tsts", "tests") self.printed_linter_info = False self.valid_messages = [] self.info_messages = [] self.warn_messages = [] self.error_messages = [] - lint_func(tool_xml, self) + lint_func(lint_target, self) # TODO: colorful emoji if in click CLI. if self.error_messages: status = "FAIL" @@ -89,3 +88,12 @@ def error(self, message, *args): def warn(self, message, *args): self.__handle_message(self.warn_messages, message, *args) + + def failed(self, fail_level): + found_warns = self.found_warns + found_errors = self.found_errors + if fail_level == LEVEL_WARN: + lint_fail = (found_warns or found_errors) + else: + lint_fail = found_errors + return lint_fail diff --git a/planemo_ext/galaxy/tools/linters/xsd.py b/planemo_ext/galaxy/tools/linters/xsd.py new file mode 100644 index 000000000..a032a7f3a --- /dev/null +++ b/planemo_ext/galaxy/tools/linters/xsd.py @@ -0,0 +1,14 @@ +from ..lint import SkipLint + +try: + from lxml import etree +except ImportError: + etree = None + + +def lint_via_xsd(tool_xml, lint_ctx): + if not lint_ctx.use_schema: + raise SkipLint() + if etree is None: + lint_ctx.error("Requested linting via XML Schema but lxml is unavailable") + return diff --git a/planemo_ext/galaxy/tools/linters/xsd_util.py b/planemo_ext/galaxy/tools/linters/xsd_util.py new file mode 100644 index 000000000..f4086ccc0 --- /dev/null +++ b/planemo_ext/galaxy/tools/linters/xsd_util.py @@ -0,0 +1,58 @@ +import abc +from collections import namedtuple + +from galaxy.tools.deps.commands import which + +try: + from lxml import etree +except ImportError: + etree = None + + +class XsdValidator(object): + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def validate(self, schema_path, target_path): + """ Validate ``target_path`` against ``schema_path``. + + :return type: ValidationResult + """ + + @abc.abstractmethod + def enabled(self): + """ Return True iff system has dependencies for this validator. + + :return type: bool + """ + +ValidationResult = namedtuple("ValidationResult", ["passed", "output"]) + + +class LxmlValidator(XsdValidator): + """ Validate XSD files using lxml library. """ + + def validate(self, schema_path, target_path): + pass + + def enabled(self): + return etree is not None + + +class XmllintValidator(XsdValidator): + """ Validate XSD files with the external tool xmllint. """ + + def validate(self, schema_path, target_path): + pass + + def enabled(self): + return bool(which("xmllint")) + + +VALIDATORS = [LxmlValidator(), XmllintValidator()] + + +def get_validator(): + for validator in VALIDATORS: + if validator.enabled(): + return validator diff --git a/scripts/update_extern.sh b/scripts/update_extern.sh index 153296a50..a6f657609 100644 --- a/scripts/update_extern.sh +++ b/scripts/update_extern.sh @@ -13,3 +13,10 @@ for file in 'rgToolFactory2.xml' 'rgToolFactory2.py' 'getlocalrpackages.py' 'LIC do wget "https://raw.githubusercontent.com/galaxyproject/tools-iuc/master/tools/tool_factory_2/$file" --output-document "$TOOL_FACTORY_PATH/$file" done + +XSD_PATH="$PLANEMO_ROOT/planemo/xml/xsd/tool" +mkdir -p $XSD_PATH +for file in 'galaxy.xsd' 'citations.xsd' 'citation.xsd' 'LICENSE'; +do + wget "https://raw.githubusercontent.com/JeanFred/Galaxy-XSD/master/$file" --output-document "$XSD_PATH/$file" +done diff --git a/setup.py b/setup.py index 9a981b36f..0c3d509ef 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,9 @@ packages=[ 'planemo', 'planemo.commands', + 'planemo.linters', 'planemo.reports', + 'planemo.xml', 'planemo_ext', 'planemo_ext.galaxy', 'planemo_ext.galaxy.eggs', @@ -62,7 +64,14 @@ 'tool_factory_2/rgToolFactory2.xml', 'tool_factory_2/rgToolFactory2.py', 'tool_factory_2/getlocalrpackages.py', - ]}, + ], + 'planemo': ['xml/xsd/repository_dependencies.xsd', + 'xml/xsd/tool_dependencies.xsd', + 'xml/xsd/tool/galaxy.xsd', + 'xml/xsd/tool/citation.xsd', + 'xml/xsd/tool/citations.xsd', + ] + }, package_dir={'planemo': 'planemo', 'planemo_ext': 'planemo_ext'}, include_package_data=True, diff --git a/tests/repository_dependencies.xml b/tests/repository_dependencies.xml new file mode 100644 index 000000000..6fd02f07a --- /dev/null +++ b/tests/repository_dependencies.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/test_utils.py b/tests/test_utils.py index 568ef53c5..f9041f7a5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,6 +8,9 @@ from planemo import cli from sys import version_info +from galaxy.tools.deps.commands import which + + if version_info < (2, 7): from unittest2 import TestCase, skip else: @@ -58,6 +61,24 @@ def skip_unless_environ(var): return skip(template % var) +def skip_unless_module(module): + available = True + try: + __import__(module) + except ImportError: + available = False + if available: + return lambda func: func + template = "Module %s could not be loaded, dependent test skipped." + return skip(template % module) + + +def skip_unless_executable(executable): + if which(executable): + return lambda func: func + return skip("PATH doesn't contain executable %s" % executable) + + __all__ = [ "TestCase", "CliTestCase", diff --git a/tests/test_validation.py b/tests/test_validation.py new file mode 100644 index 000000000..4d54b0bff --- /dev/null +++ b/tests/test_validation.py @@ -0,0 +1,66 @@ +import os + +from .test_utils import ( + skip_unless_module, + skip_unless_executable, +) + +from planemo.xml import validation +from planemo import shed_lint + +TEST_PATH = os.path.dirname(__file__) + + +@skip_unless_module("lxml") +def test_lxml_validation(): + lxml_xsd_validator = validation.LxmlValidator() + _check_validator(lxml_xsd_validator) + + +@skip_unless_executable("xmllint") +def test_xmllint_validation(): + xmllint_xsd_validator = validation.XmllintValidator() + _check_validator(xmllint_xsd_validator) + + +def test_tool_dependencies_validation(): + _assert_validates(shed_lint.TOOL_DEPENDENCIES_XSD, + _path("tool_dependencies_good_1.xml")) + _assert_validates(shed_lint.TOOL_DEPENDENCIES_XSD, + _path("tool_dependencies_good_2.xml")) + + +def test_repository_dependencies_validation(): + _assert_validates(shed_lint.REPO_DEPENDENCIES_XSD, + _path("repository_dependencies.xml")) + + +def _check_validator(xsd_validator): + _assert_validates( + _path("xsd_schema_1.xsd"), + _path("xml_good_1.xml"), + xsd_validator, + ) + _assert_validates( + _path("xsd_schema_1.xsd"), + _path("xml_good_2.xml"), + xsd_validator, + ) + + result = xsd_validator.validate(_path("xsd_schema_1.xsd"), + _path("xml_bad_1.xml")) + assert not result.passed + output = result.output + assert "not_command" in str(output), str(output) + + +def _assert_validates(schema, target, xsd_validator=None): + if xsd_validator is None: + xsd_validator = validation.get_validator() + result = xsd_validator.validate(schema, + target) + assert result.passed, result.output + + +def _path(filename): + return os.path.join(TEST_PATH, filename) diff --git a/tests/tool_dependencies_good_1.xml b/tests/tool_dependencies_good_1.xml new file mode 100644 index 000000000..7dfe8f1e3 --- /dev/null +++ b/tests/tool_dependencies_good_1.xml @@ -0,0 +1,82 @@ + + + + + + + http://depot.galaxyproject.org/package/linux/i386/samtools/samtools-0.1.16-linux-i386.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/linux/x86_64/samtools/samtools-0.1.16-linux-x86_64.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/darwin/i386/samtools/samtools-0.1.16-Darwin-i386.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/darwin/x86_64/samtools/samtools-0.1.16-Darwin-x86_64.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/source/samtools/samtools-0.1.16.tar.bz2 + sed -i.bak 's/-lcurses/-lncurses/' Makefile + make + + samtools + $INSTALL_DIR/bin + + + libbam.a + $INSTALL_DIR/lib + + + + $INSTALL_DIR/bin + $INSTALL_DIR/lib + + + + + This is the last version of SAMTools to include the 'pileup' command. + + Program: samtools (Tools for alignments in the SAM format) + Version: 0.1.16 (r963:234) + + Usage: samtools <command> [options] + + Command: view SAM<->BAM conversion + sort sort alignment file + pileup generate pileup output + mpileup multi-way pileup + depth compute the depth + faidx index/extract FASTA + tview text alignment viewer + index index alignment + idxstats BAM index stats (r595 or later) + fixmate fix mate information + glfview print GLFv3 file + flagstat simple stats + calmd recalculate MD/NM tags and '=' bases + merge merge sorted alignments + rmdup remove PCR duplicates + reheader replace BAM header + cat concatenate BAMs + targetcut cut fosmid regions (for fosmid pool only) + phase phase heterozygotes + + + diff --git a/tests/tool_dependencies_good_2.xml b/tests/tool_dependencies_good_2.xml new file mode 100644 index 000000000..acee07390 --- /dev/null +++ b/tests/tool_dependencies_good_2.xml @@ -0,0 +1,102 @@ + + + + + + + http://depot.galaxyproject.org/package/darwin/x86_64/R/R-2.15.0-Darwin-x86_64.tgz + + . + $INSTALL_DIR + + + + http://depot.galaxyproject.org/package/linux/x86_64/R/R-2.15.0-Linux-x86_64.tgz + + . + $INSTALL_DIR + + + $INSTALL_DIR/lib/libtcl8.4.so + $INSTALL_DIR/lib/libtk8.4.so + + + + + + + + + + + + + + + + + + + http://cran.rstudio.com/src/base/R-2/R-2.15.0.tar.gz + + + + + + + + + + + + + + + + + + + export LDFLAGS="-L$PNG_LIBS -L$READLINE_LIBS" && + export LDFLAGS="$LDFLAGS -Wl,-rpath,$READLINE_LIBS" && + export LDFLAGS="$LDFLAGS -Wl,-rpath,$PIXMAN_LIB_PATH" && + export LDFLAGS="$LDFLAGS -Wl,-rpath,$PNG_LIB_PATH" && + export LDFLAGS="$LDFLAGS -Wl,-rpath,$FREETYPE_LIB_PATH" && + export CFLAGS="-I$PNG_INCLUDES -I$READLINE_INCLUDES" && + export CXXFLAGS="-I$PNG_INCLUDES -I$READLINE_INCLUDES" && + export CPPFLAGS="-I$PNG_INCLUDES -I$READLINE_INCLUDES" && + ./configure --with-tcltk \ + --with-blas \ + --with-lapack \ + --with-readline \ + --with-cairo \ + --with-libpng \ + --without-x \ + --enable-R-shlib \ + --disable-R-framework \ + --libdir=$INSTALL_DIR/lib \ + --prefix=$INSTALL_DIR + + + sed -i.bak 's;$INSTALL_DIR;\${R_ROOT_DIR};g' $INSTALL_DIR/bin/R + sed -i.bak 's;$INSTALL_DIR;\${R_ROOT_DIR};g' $INSTALL_DIR/lib/R/bin/R + + + $INSTALL_DIR/lib/R/doc + $INSTALL_DIR/lib/R + $INSTALL_DIR/lib/R/include + $INSTALL_DIR/lib/R/share + $INSTALL_DIR + $INSTALL_DIR/lib/R + $INSTALL_DIR/lib/R + $INSTALL_DIR/lib/R/library + $INSTALL_DIR/bin + + + + + The precompiled versions of this package were built 2014-03-06. + R is a free software environment for statistical computing and graphics. + NOTE: See custom compilation options above. + + + \ No newline at end of file diff --git a/tests/xml_bad_1.xml b/tests/xml_bad_1.xml new file mode 100644 index 000000000..82d6fc40e --- /dev/null +++ b/tests/xml_bad_1.xml @@ -0,0 +1,2 @@ + + diff --git a/tests/xml_good_1.xml b/tests/xml_good_1.xml new file mode 100644 index 000000000..f3d3141c9 --- /dev/null +++ b/tests/xml_good_1.xml @@ -0,0 +1,2 @@ + + diff --git a/tests/xml_good_2.xml b/tests/xml_good_2.xml new file mode 100644 index 000000000..94cd80141 --- /dev/null +++ b/tests/xml_good_2.xml @@ -0,0 +1,3 @@ + + moo.py --arg 1 + diff --git a/tests/xsd_command.xsd b/tests/xsd_command.xsd new file mode 100644 index 000000000..283601042 --- /dev/null +++ b/tests/xsd_command.xsd @@ -0,0 +1,23 @@ + + + + + + This tag specifies how Galaxy should invoke the tool's executable, passing its required input parameter values (the command line specification links the parameters supplied in the form with the actual tool executable). Any word inside it starting with a dollar sign ($) will be treated as a variable whose values can be acquired from one of three sources: parameters, metadata, or output files. After the substitution of variables with their values, the content is interpreted with Cheetah and finally given to the interpreter specified in the corresponding attribute (if any). + + + + + + This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. + + + + + + + \ No newline at end of file diff --git a/tests/xsd_schema_1.xsd b/tests/xsd_schema_1.xsd new file mode 100644 index 000000000..24696e8d8 --- /dev/null +++ b/tests/xsd_schema_1.xsd @@ -0,0 +1,9 @@ + + + + + \ No newline at end of file From 5205284c922980b701a2e74b7d343ad6f70cbc6b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 20 Apr 2015 18:31:23 -0400 Subject: [PATCH 2/2] Rebase galaxy.xsd et. al. as submodules. --- .gitmodules | 3 + planemo/xml/xsd/tool | 1 + planemo/xml/xsd/tool/LICENSE | 22 - planemo/xml/xsd/tool/citation.xsd | 30 - planemo/xml/xsd/tool/citations.xsd | 15 - planemo/xml/xsd/tool/galaxy.xsd | 1277 ---------------------------- scripts/update_extern.sh | 7 - 7 files changed, 4 insertions(+), 1351 deletions(-) create mode 100644 .gitmodules create mode 160000 planemo/xml/xsd/tool delete mode 100644 planemo/xml/xsd/tool/LICENSE delete mode 100644 planemo/xml/xsd/tool/citation.xsd delete mode 100644 planemo/xml/xsd/tool/citations.xsd delete mode 100644 planemo/xml/xsd/tool/galaxy.xsd diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..7a2486b41 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "planemo/xml/xsd/tool"] + path = planemo/xml/xsd/tool + url = https://github.com/JeanFred/Galaxy-XSD diff --git a/planemo/xml/xsd/tool b/planemo/xml/xsd/tool new file mode 160000 index 000000000..ae723a187 --- /dev/null +++ b/planemo/xml/xsd/tool @@ -0,0 +1 @@ +Subproject commit ae723a1873bd6a249ba60b735dbbbb02e8427dad diff --git a/planemo/xml/xsd/tool/LICENSE b/planemo/xml/xsd/tool/LICENSE deleted file mode 100644 index bfa63c52c..000000000 --- a/planemo/xml/xsd/tool/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2011-2015 John Chilton, Jean-Frédéric Berthelot, Pierre Lindenbaum - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - diff --git a/planemo/xml/xsd/tool/citation.xsd b/planemo/xml/xsd/tool/citation.xsd deleted file mode 100644 index 81c470687..000000000 --- a/planemo/xml/xsd/tool/citation.xsd +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. - - - - - - type of citation - - - - - - - - - Type of citation represented. - - - - - - - - diff --git a/planemo/xml/xsd/tool/citations.xsd b/planemo/xml/xsd/tool/citations.xsd deleted file mode 100644 index 3db22e7c9..000000000 --- a/planemo/xml/xsd/tool/citations.xsd +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - Tool files may declare one citations element. Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. - - - - - - - - diff --git a/planemo/xml/xsd/tool/galaxy.xsd b/planemo/xml/xsd/tool/galaxy.xsd deleted file mode 100644 index fbcde8349..000000000 --- a/planemo/xml/xsd/tool/galaxy.xsd +++ /dev/null @@ -1,1277 +0,0 @@ - - - - Galaxy Schema - A Galaxy XML tool wrapper - - - - - - - - The outer-most tag set - - - - - - - The value is displayed in the tool menu immediately following the hyperlink for the tool - - - - - - Specifies the command to be run in order to get the tool's version string. The resulting value will be found in the "Info" field of the history dataset. - - - - - - - - - - - Documentation for help - - - - - - - - - Must be unique across all tools; should be lowercase and contain only letters, numbers, and underscores. It allows for tool versioning and metrics of the number of times a tool is used, among other things. - - - - - This string is what is displayed as a hyperlink in the tool menu - - - - - This string defaults to "1.0.0' if it is not included in the tag. It allows for tool versioning and should be changed with each new version of the tool. - - - - - Allows for tools to be loaded upon server startup, but not displayed in the tool menu - - - - - Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. - - - - - Only if "tool_type" attribute value is "data_source" - defines the HTTP request method to use when communicating with an external data source application ( the default is "get" ). - - - - - - - - This is a container tag set for the 'requirement' tag set described below. - - - - - - - - - - This tag set is contained within the 'requirements' tag set. Third party programs or modules that the tool depends upon (and which are not distributed with Galaxy) are included in this tag set. The intention is that when Galaxy starts it can check whether the required programs or modules are available, and if not this tool will not be loaded. - - - - - - This value defines the which type of the 3rd party module required by this tool - - - - - Required for package type requirements - - - - - - - - This tag set is contained within the 'requirements' tag set. Galaxy can be configured to run tools within Docker (https://www.docker.com/) containers - this tag allows the tool to suggest possible valid Docker containers for this tool. - - - - - - - This value describes the type of container that the tool may be executed in and currently must be 'docker'. - - - - - - - - - Documentation for Parallelism - - - - Documentation for method - - - - - - Documentation for Code - - - - Documentation for file - - - - - - Documentation for UIhints - - - - Documentation for minwidth - - - - - - Documentation for Tests - - - - - - - - Documentation for Test - - - - - - - - - Documentation for TestParam - - - - - - - Documentation for name - - - - - Documentation for value - - - - - Documentation for ftype - - - - - Documentation for dbkey - - - - - - Definition of a collection for test input. - - - - - - - Type of collection to create. - - - - - - Documentation for TestOutput - - - - - - - Documentation for name - - - - - Documentation for file - - - - - Documentation for ftype - - - - - Documentation for sort - - - - - - Documentation for TestAssertion - - - - - - - - - - - - - - - Line the test output must contain. - - - - - - Consists of all tag sets that define the tool's input parameters. Each <param> tag within the <inputs> tag set maps to a command line parameter within the 'command' tag set described above. - - - - - - - Documentation for action - - - - - Documentation for check_values - - - - - Documentation for method - - - - - Documentation for target - - - - - - - - - - - - - Documentation for display - - - - - - - Documentation for InputType - - - - - Name for this element - - - - - - - - This is a container for conditional parameters in the tool ( must contain 'when' tag sets ) - the command line is wrapped in an if-else statement. - - - - - - - - - - - - - - - - Documentation for ConditionalWhen - - - - - - - Documentation for value - - - - - - - - - This is a container for any tag sets that can be contained within the 'inputs' tag set. When this is used, the tool will allow the user to add any number of additional sets of the contained parameters ( an "Add new 'title'" button will be displayed on the tool form ). - - - - - - - - - The title of the repeat section, which will be displayed on the tool form - - - - - The minimum number of repeat units - - - - - The maximum number of repeat units - - - - - The default number of repeat units - - - - - - - - - - - Documentation for Param - - - - - - - Documentation for label - - - - - - - - - - - - Documentation for type - - - - - Documentation for label - - - - - Documentation for help - - - - - Documentation for value - - - - - Documentation for optional - - - - - Documentation for min - - - - - Documentation for max - - - - - Documentation for format - - - - - Restrict valid collection types for data_collection params (e.g. list, paired, list:paired, etc...). - - - - - Documentation for data_ref - - - - - Documentation for accept_default - - - - - Documentation for force_select - - - - - Documentation for display - - - - - Documentation for multiple - - - - - Documentation for numerical - - - - - Documentation for hierarchy - - - - - Documentation for checked - - - - - Documentation for truevalue - - - - - Documentation for falsevalue - - - - - Documentation for size - - - - - Documentation for metadata_name - - - - - - - - Documentation for ParamType - - - - - - - - - - - - - - - - - - - - - - This tag specifies how Galaxy should invoke the tool's executable, passing its required input parameter values (the command line specification links the parameters supplied in the form with the actual tool executable). Any word inside it starting with a dollar sign ($) will be treated as a variable whose values can be acquired from one of three sources: parameters, metadata, or output files. After the substitution of variables with their values, the content is interpreted with Cheetah and finally given to the interpreter specified in the corresponding attribute (if any). - - - - - - This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. - - - - - - - - - - Documentation for ParamOption - - - - - - Documentation for value - - - - - Documentation for selected - - - - - - - - - - Documentation for Options - - - - - - - Documentation for from_dataset - - - - - Documentation for from_file - - - - - Documentation for from_data_table - - - - - Documentation for from_parameter - - - - - Documentation for options_filter_attribute - - - - - Documentation for transform_lines - - - - - Documentation for startswith - - - - - - - - - - - Documentation for file - - - - - - - Documentation for Column - - - - Documentation for name - - - - - Documentation for index - - - - - - Documentation for Validator - - - - Documentation for type - - - - - Documentation for message - - - - - Documentation for filename - - - - - Documentation for metadata_name - - - - - Documentation for metadata_column - - - - - Documentation for line_startswith - - - - - Documentation for min - - - - - Documentation for max - - - - - Documentation for split - - - - - - Documentation for Sanitizer - - - - - - - Documentation for sanitize - - - - - Documentation for invalid_char - - - - - - Documentation for Valid - - - - - - - Documentation for initial - - - - - - Documentation for ValidElementType - - - - Documentation for preset - - - - - Documentation for value - - - - - - Documentation for Add - - - - - - - - Documentation for Remove - - - - - - - - - - - - - - Documentation for Filter - - - - Documentation for type - - - - - Documentation for name - - - - - Documentation for column - - - - - Documentation for ref - - - - - Documentation for key - - - - - Documentation for separator - - - - - Documentation for compare - - - - - Documentation for keep - - - - - Documentation for value - - - - - Documentation for ref_attribute - - - - - - Documentation for Outputs - - - - - - - - Documentation for Data - - - - - - Documentation for filter - - - - - - - Documentation for name - - - - - Documentation for format - - - - - Documentation for format_source - - - - - Documentation for metadata_source - - - - - Documentation for label - - - - - Documentation for from_work_dir - - - - - Documentation for hidden - - - - - - Documentation for Actions - - - - - - - - - - - - - - Documentation for Action - - - - - - - Documentation for type - - - - - Documentation for name - - - - - Documentation for default - - - - - - Documentation for ActionsOption - - - - - - - Documentation for type - - - - - Documentation for name - - - - - Documentation for column - - - - - Documentation for offset - - - - - Documentation for param_attribute - - - - - - Documentation for ActionsConditional - - - - - - - - - - - - Documentation for ActionsConditionalWhen - - - - - - - Documentation for value - - - - - - Documentation for ConfigFiles - - - - - - - - Documentation for ConfigFile - - - - - - Documentation for name - - - - - Documentation for type - - - - - - - - - - Galaxy currently supports using regular expressions to scan stdout and stderr, and it also allows exit codes to be scanned for ranges. The 'stdio' tag has two subtags, 'regex' and 'exit_code', to define regular expressions and exit code processing, respectively. - - - - - - - - - - - Documentation for ExitCode - - - - Documentation for range - - - - - Documentation for level - - - - - Documentation for description - - - - - - - - A regular expression defines a pattern of characters - - - - This tells whether the regular expression should be matched against stdout, stderr, or both. If this attribute is missing or is incorrect, then both stdout and stderr will be checked. The source can be one of the follwing values: - - - - - This is the regular expression that will be used to match against stdout and/or stderr. - - - - - This works very similarly to the 'exit_code' tag, except that, when a regular expression matches against its source, the description is added to the beginning of the source. - - - - - an optional description of the regular expression that has matched. - - - - - - - - Documentation for ConfigFileType - - - - - - - - - - Documentation for ChangeFormat - - - - - - - - Documentation for WhenData - - - - - Documentation for input - - - - - Documentation for value - - - - - Documentation for format - - - - - - Documentation for RequirementType - - - - - - - - - - Type of container for tool execution. - - - - - - - - Documentation for ToolTypeType - - - - - - - - Documentation for URLmethodType - - - - - - - - - Documentation for TargetType - - - - - - - - - Documentation for MethodType - - - - - - - - - Documentation for DisplayType - - - - - - - - - Documentation for HierarchyType - - - - - - - - - Documentation for CheckedType - - - - - - - - - - - - - Documentation for ValidatorType - - - - - - - - - - - - - - - - - Documentation for FilterType - - - - - - - - - - - - - - Documentation for ActionType - - - - - - - - - Documentation for ActionsOptionType - - - - - - - - - - Documentation for CompareType - - - - - - - - Documentation for LevelType - - - - - - - - - - Documentation for RangeType - - - - - - - - Documentation for SourceType - - - - - - - - - - Documentation for PermissiveBoolean - - - - - - - - - - - diff --git a/scripts/update_extern.sh b/scripts/update_extern.sh index a6f657609..153296a50 100644 --- a/scripts/update_extern.sh +++ b/scripts/update_extern.sh @@ -13,10 +13,3 @@ for file in 'rgToolFactory2.xml' 'rgToolFactory2.py' 'getlocalrpackages.py' 'LIC do wget "https://raw.githubusercontent.com/galaxyproject/tools-iuc/master/tools/tool_factory_2/$file" --output-document "$TOOL_FACTORY_PATH/$file" done - -XSD_PATH="$PLANEMO_ROOT/planemo/xml/xsd/tool" -mkdir -p $XSD_PATH -for file in 'galaxy.xsd' 'citations.xsd' 'citation.xsd' 'LICENSE'; -do - wget "https://raw.githubusercontent.com/JeanFred/Galaxy-XSD/master/$file" --output-document "$XSD_PATH/$file" -done