-
Notifications
You must be signed in to change notification settings - Fork 85
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #130 from galaxyproject/xsd
Use XSD to lint tools and repositories.
- Loading branch information
Showing
31 changed files
with
1,081 additions
and
72 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
[submodule "planemo/xml/xsd/tool"] | ||
path = planemo/xml/xsd/tool | ||
url = https://github.com/JeanFred/Galaxy-XSD |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,55 +1,24 @@ | ||
import sys | ||
import traceback | ||
|
||
import click | ||
|
||
from planemo.cli import pass_context | ||
from planemo.io import info | ||
from planemo.io import error | ||
from planemo import options | ||
|
||
from galaxy.tools.loader_directory import load_tool_elements_from_path | ||
from galaxy.tools.lint import lint_xml | ||
|
||
SKIP_XML_MESSAGE = "Skipping XML file - does not appear to be a tool %s." | ||
LINTING_TOOL_MESSAGE = "Linting tool %s" | ||
from planemo.tool_lint import build_lint_args | ||
from planemo.tool_lint import lint_tools_on_path | ||
|
||
|
||
@click.command('lint') | ||
@options.optional_tools_arg() | ||
@click.option( | ||
'--report_level', | ||
type=click.Choice(['all', 'warn', 'error']), | ||
default="all" | ||
) | ||
@click.option( | ||
'--fail_level', | ||
type=click.Choice(['warn', 'error']), | ||
default="warn" | ||
) | ||
@options.report_level_option() | ||
@options.fail_level_option() | ||
@options.lint_xsd_option() | ||
@pass_context | ||
def cli(ctx, path, report_level="all", fail_level="warn"): | ||
def cli(ctx, path, **kwds): | ||
"""Check specified tool(s) for common errors and adherence to best | ||
practices. | ||
""" | ||
exit = 0 | ||
lint_args = dict(level=report_level, fail_level=fail_level) | ||
tools = load_tool_elements_from_path(path, load_exception_handler) | ||
valid_tools = 0 | ||
for (tool_path, tool_xml) in tools: | ||
if tool_xml.getroot().tag != "tool": | ||
if ctx.verbose: | ||
info(SKIP_XML_MESSAGE % tool_path) | ||
continue | ||
info("Linting tool %s" % tool_path) | ||
if not lint_xml(tool_xml, **lint_args): | ||
exit = 1 | ||
else: | ||
valid_tools += 1 | ||
if exit == 0 and valid_tools == 0: | ||
exit = 2 | ||
lint_args = build_lint_args(**kwds) | ||
exit = lint_tools_on_path(ctx, path, lint_args) | ||
sys.exit(exit) | ||
|
||
|
||
def load_exception_handler(path, exc_info): | ||
error("Error loading tool with path %s" % path) | ||
traceback.print_exception(*exc_info, limit=1, file=sys.stderr) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
import click | ||
import sys | ||
|
||
from planemo.cli import pass_context | ||
from planemo import options | ||
from planemo import shed | ||
from planemo import shed_lint | ||
|
||
|
||
@click.command('shed_lint') | ||
@options.optional_project_arg(exists=True) | ||
@options.report_level_option() | ||
@options.fail_level_option() | ||
@options.click.option( | ||
'--tools', | ||
is_flag=True, | ||
default=False, | ||
help=("Lint tools discovered in the process of linting repositories.") | ||
) | ||
@options.lint_xsd_option() | ||
@options.recursive_shed_option() | ||
@pass_context | ||
def cli(ctx, path, recursive=False, **kwds): | ||
"""Check a Tool Shed repository for common problems. | ||
""" | ||
def lint(path): | ||
return shed_lint.lint_repository(ctx, path, **kwds) | ||
|
||
if recursive: | ||
exit_code = shed.for_each_repository(lint, path) | ||
else: | ||
exit_code = lint(path) | ||
|
||
sys.exit(exit_code) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
import os | ||
|
||
from planemo.xml import validation | ||
|
||
|
||
def lint_xsd(lint_ctx, schema_path, path): | ||
name = os.path.basename(path) | ||
validator = validation.get_validator(require=True) | ||
validation_result = validator.validate(schema_path, path) | ||
if not validation_result.passed: | ||
msg = "Invalid %s found. Errors [%s]" | ||
msg = msg % (name, validation_result.output) | ||
lint_ctx.error(msg) | ||
else: | ||
lint_ctx.info("%s found and appears to be valid XML" % name) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
""" Tool linting module that lints Galaxy tool against experimental XSD. | ||
""" | ||
import copy | ||
import os | ||
import tempfile | ||
|
||
from planemo.xml import XSDS_PATH | ||
import planemo.lint | ||
|
||
TOOL_XSD = os.path.join(XSDS_PATH, "tool", "galaxy.xsd") | ||
|
||
|
||
def lint_tool_xsd(root, lint_ctx): | ||
""" Write a temp file out and lint it. | ||
""" | ||
with tempfile.NamedTemporaryFile() as tf: | ||
_clean_root(root).write(tf.name) | ||
planemo.lint.lint_xsd(lint_ctx, TOOL_XSD, tf.name) | ||
|
||
|
||
def _clean_root(root): | ||
""" XSD assumes macros have been expanded, so remove them. | ||
""" | ||
clean_root = copy.deepcopy(root) | ||
to_remove = [] | ||
for macros_el in clean_root.findall("macros"): | ||
to_remove.append(macros_el) | ||
for macros_el in to_remove: | ||
clean_root.getroot().remove(macros_el) | ||
return clean_root |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
import os | ||
import yaml | ||
from galaxy.tools.lint import LintContext | ||
from planemo.lint import lint_xsd | ||
from planemo.tool_lint import ( | ||
build_lint_args, | ||
yield_tool_xmls, | ||
) | ||
from planemo.xml import XSDS_PATH | ||
|
||
|
||
from planemo.io import info | ||
from planemo.io import error | ||
|
||
from galaxy.tools.lint import lint_xml_with | ||
|
||
TOOL_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "tool_dependencies.xsd") | ||
REPO_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "repository_dependencies.xsd") | ||
|
||
|
||
def lint_repository(ctx, path, **kwds): | ||
info("Linting repository %s" % path) | ||
lint_args = build_lint_args(**kwds) | ||
lint_ctx = LintContext(lint_args["level"]) | ||
lint_ctx.lint( | ||
"tool_dependencies", | ||
lint_tool_dependencies, | ||
path, | ||
) | ||
lint_ctx.lint( | ||
"repository_dependencies", | ||
lint_repository_dependencies, | ||
path, | ||
) | ||
lint_ctx.lint( | ||
"shed_yaml", | ||
lint_shed_yaml, | ||
path, | ||
) | ||
if kwds["tools"]: | ||
for (tool_path, tool_xml) in yield_tool_xmls(ctx, path): | ||
info("+Linting tool %s" % tool_path) | ||
lint_xml_with( | ||
lint_ctx, | ||
tool_xml, | ||
extra_modules=lint_args["extra_modules"] | ||
) | ||
failed = lint_ctx.failed(lint_args["fail_level"]) | ||
if failed: | ||
error("Failed linting") | ||
return 1 if failed else 0 | ||
|
||
|
||
def lint_tool_dependencies(path, lint_ctx): | ||
tool_dependencies = os.path.join(path, "tool_dependencies.xml") | ||
if not os.path.exists(tool_dependencies): | ||
lint_ctx.info("No tool_dependencies.xml, skipping.") | ||
return | ||
lint_xsd(lint_ctx, TOOL_DEPENDENCIES_XSD, tool_dependencies) | ||
|
||
|
||
def lint_repository_dependencies(path, lint_ctx): | ||
repo_dependencies = os.path.join(path, "repository_dependencies.xml") | ||
if not os.path.exists(repo_dependencies): | ||
lint_ctx.info("No repository_dependencies.xml, skipping.") | ||
return | ||
lint_xsd(lint_ctx, REPO_DEPENDENCIES_XSD, repo_dependencies) | ||
|
||
|
||
def lint_shed_yaml(path, lint_ctx): | ||
shed_yaml = os.path.join(path, ".shed.yml") | ||
if not os.path.exists(shed_yaml): | ||
lint_ctx.info("No .shed.yml file found, skipping.") | ||
return | ||
try: | ||
shed_contents = yaml.load(open(shed_yaml, "r")) | ||
except Exception as e: | ||
lint_ctx.warn("Failed to parse .shed.yml file [%s]" % str(e)) | ||
|
||
warned = False | ||
for required_key in ["owner", "name"]: | ||
if required_key not in shed_contents: | ||
lint_ctx.warn(".shed.yml did not contain key [%s]" % required_key) | ||
warned = True | ||
|
||
if not warned: | ||
lint_ctx.info(".shed.yml found and appears to be valid YAML.") |
Oops, something went wrong.