Merge pull request #3654 from jmchilton/galaxy-lib-update

Update against the latest changes in galaxy-lib

lib/galaxy/exceptions/error_codes.py

@@ -3,6 +3,7 @@
 See the file error_codes.json for actual error code descriptions.
 """
 from json import loads
+
 from pkg_resources import resource_string
 
 

lib/galaxy/tools/deps/conda_util.py

@@ -249,16 +249,18 @@ def exec_install(self, args):
         install_base_args.extend(args)
         return self.exec_command("install", install_base_args)
 
-    def exec_clean(self, args=[]):
+    def exec_clean(self, args=[], quiet=False):
         """
         Clean up after conda installation.
         """
         clean_base_args = [
             "--tarballs",
             "-y"
         ]
-        clean_base_args.extend(args)
-        return self.exec_command("clean", clean_base_args)
+        clean_args = clean_base_args + args
+        if quiet:
+            clean_args.extend([">", "/dev/null"])
+        return self.exec_command("clean", clean_args)
 
     def export_list(self, name, path):
         return self.exec_command("list", [
@@ -400,27 +402,33 @@ def install_conda(conda_context=None):
             os.remove(script_path)
 
 
-def install_conda_targets(conda_targets, env_name, conda_context=None):
+def install_conda_targets(conda_targets, env_name=None, conda_context=None):
     conda_context = _ensure_conda_context(conda_context)
     conda_context.ensure_channels_configured()
-    create_args = [
-        "--name", env_name,  # enviornment for package
-    ]
-    for conda_target in conda_targets:
-        create_args.append(conda_target.package_specifier)
-    return conda_context.exec_create(create_args)
+    if env_name is not None:
+        create_args = [
+            "--name", env_name,  # environment for package
+        ]
+        for conda_target in conda_targets:
+            create_args.append(conda_target.package_specifier)
+        return conda_context.exec_create(create_args)
+    else:
+        return conda_context.exec_install([t.package_specifier for t in conda_targets])
 
 
-def install_conda_target(conda_target, conda_context=None):
+def install_conda_target(conda_target, conda_context=None, skip_environment=False):
     """ Install specified target into a its own environment.
     """
     conda_context = _ensure_conda_context(conda_context)
     conda_context.ensure_channels_configured()
-    create_args = [
-        "--name", conda_target.install_environment,  # enviornment for package
-        conda_target.package_specifier,
-    ]
-    return conda_context.exec_create(create_args)
+    if not skip_environment:
+        create_args = [
+            "--name", conda_target.install_environment,  # environment for package
+            conda_target.package_specifier,
+        ]
+        return conda_context.exec_create(create_args)
+    else:
+        return conda_context.exec_install([conda_target.package_specifier])
 
 
 def cleanup_failed_install_of_environment(env, conda_context=None):
@@ -507,6 +515,7 @@ def build_isolated_environment(
     path=None,
     copy=False,
     conda_context=None,
+    quiet=False,
 ):
     """ Build a new environment (or reuse an existing one from hashes)
     for specified conda packages.
@@ -552,14 +561,17 @@ def build_isolated_environment(
                 "--file", export_path, ">", "/dev/null"
             ])
 
+        if quiet:
+            create_args.extend([">", "/dev/null"])
+
         if path is not None and os.path.exists(path):
             exit_code = conda_context.exec_install(create_args)
         else:
             exit_code = conda_context.exec_create(create_args)
 
         return (path or tempdir_name, exit_code)
     finally:
-        conda_context.exec_clean()
+        conda_context.exec_clean(quiet=quiet)
         shutil.rmtree(tempdir)
 
 

lib/galaxy/tools/deps/mulled/invfile.lua

@@ -34,6 +34,12 @@ for i = 1, #binds_table do
     table.insert(bind_args, binds_table[i])
 end
 
+local test_bind_args = {}
+local test_binds_table = VAR.TEST_BINDS:split(",")
+for i = 1, #test_binds_table do
+    table.insert(test_bind_args, test_binds_table[i])
+end
+
 inv.task('build')
     .using('continuumio/miniconda:latest')
         .withHostConfig({binds = {"build:/data"}})
@@ -49,10 +55,18 @@ inv.task('build')
         .inImage('bgruening/busybox-bash:0.1')
         .as(repo)
 
-inv.task('test')
-    .using(repo)
-    .withConfig({entrypoint = {'/bin/sh', '-c'}})
-    .run(VAR.TEST)
+if VAR.TEST_BINDS == '' then
+    inv.task('test')
+        .using(repo)
+        .withConfig({entrypoint = {'/bin/sh', '-c'}})
+        .run(VAR.TEST)
+else
+    inv.task('test')
+        .using(repo)
+        .withHostConfig({binds = test_bind_args})
+        .withConfig({entrypoint = {'/bin/sh', '-c'}})
+        .run(VAR.TEST)
+end
 
 inv.task('push')
     .push(repo)

lib/galaxy/tools/deps/mulled/mulled_build.py

@@ -34,6 +34,7 @@
 DEFAULT_CHANNELS = [DEFAULT_CHANNEL] + DEFAULT_EXTRA_CHANNELS
 DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}"
 DEFAULT_BINDS = ["build/dist:/usr/local/"]
+DEFAULT_WORKING_DIR = '/source/'
 IS_OS_X = _platform == "darwin"
 INVOLUCRO_VERSION = "1.1.2"
 
@@ -113,7 +114,7 @@ def conda_versions(pkg_name, file_name):
 def mull_targets(
     targets, involucro_context=None,
     command="build", channels=DEFAULT_CHANNELS, namespace="mulled",
-    test='true', image_build=None, name_override=None,
+    test='true', test_files=None, image_build=None, name_override=None,
     repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
     binds=DEFAULT_BINDS
 ):
@@ -144,6 +145,18 @@ def mull_targets(
         '-set', "BINDS='%s'" % bind_str,
         command,
     ]
+    if test_files:
+        test_bind = []
+        for test_file in test_files:
+            if ':' not in test_file:
+                if os.path.exists(test_file):
+                    test_bind.append("%s:%s/%s" % (test_file, DEFAULT_WORKING_DIR, test_file))
+            else:
+                if os.path.exists(test_file.split(':')[0]):
+                    test_bind.append(test_file)
+        if test_bind:
+            involucro_args.insert(6, '-set')
+            involucro_args.insert(7, "TEST_BINDS='%s'" % ",".join(test_bind))
     print(" ".join(involucro_context.build_command(involucro_args)))
     if not dry_run:
         ensure_installed(involucro_context, True)
@@ -252,6 +265,9 @@ def args_to_mull_targets_kwds(args):
         kwds["dry_run"] = args.dry_run
     if hasattr(args, "test"):
         kwds["test"] = args.test
+    if hasattr(args, "test_files"):
+        if args.test_files:
+            kwds["test_files"] = args.test_files.split(",")
     if hasattr(args, "channel"):
         channels = [args.channel]
         if hasattr(args, "extra_channels"):
@@ -276,6 +292,8 @@ def main(argv=None):
     parser.add_argument('targets', metavar="TARGETS", default=None, help="Build a single container with specific package(s).")
     parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of mulled container (leave blank to auto-generate based on packages - recommended).")
     parser.add_argument('--test', help='Provide a test command for the container.')
+    parser.add_argument('--test-files', help='Provide test-files that may be required to run the test command. Individual mounts are separated by comma.'
+                                             'The source:dest docker syntax is respected. If relative file paths are given, files will be mounted in /source/<relative_file_path>')
     args = parser.parse_args()
     targets = target_str_to_targets(args.targets)
     sys.exit(mull_targets(targets, **args_to_mull_targets_kwds(args)))

lib/galaxy/tools/linters/general.py

@@ -10,6 +10,11 @@
 ERROR_ID_MSG = "Tool does not define an id attribute."
 VALID_ID_MSG = "Tool defines an id [%s]."
 
+PROFILE_PATTERN = re.compile(r"^[1,2]\d\.[0,1]\d$")
+PROFILE_INFO_DEFAULT_MSG = "Tool targets 16.01 Galaxy profile."
+PROFILE_INFO_SPECIFIED_MSG = "Tool specifies profile version [%s]."
+PROFILE_INVALID_MSG = "Tool specifies an invalid profile version [%s]."
+
 lint_tool_types = ["*"]
 
 
@@ -33,5 +38,14 @@ def lint_general(tool_source, lint_ctx):
     else:
         lint_ctx.valid(VALID_ID_MSG % tool_id)
 
+    profile = tool_source.parse_profile()
+    profile_valid = PROFILE_PATTERN.match(profile) is not None
+    if not profile_valid:
+        lint_ctx.warn(PROFILE_INVALID_MSG)
+    elif profile == "16.01":
+        lint_ctx.valid(PROFILE_INFO_DEFAULT_MSG)
+    else:
+        lint_ctx.valid(PROFILE_INFO_SPECIFIED_MSG % profile)
+
     if re.search(r"\s", tool_id):
         lint_ctx.warn("Tool id contains a space - this is discouraged.")

lib/galaxy/tools/linters/outputs.py

@@ -20,6 +20,9 @@ def lint_output(tool_xml, lint_ctx):
             continue
         num_outputs += 1
         output_attrib = output.attrib
+        if "name" not in output_attrib:
+            lint_ctx.warn("Tool output doesn't define a name - this is likely a problem.")
+
         if output.tag == "data":
             format_set = False
             if "format" in output_attrib:

lib/galaxy/tools/linters/stdio.py

@@ -2,12 +2,17 @@
 from .command import get_command
 
 
-def lint_stdio(tool_xml, lint_ctx):
-    stdios = tool_xml.findall("./stdio")
+def lint_stdio(tool_source, lint_ctx):
+    tool_xml = getattr(tool_source, "xml_tree", None)
+    stdios = tool_xml.findall("./stdio") if tool_xml else []
+
     if not stdios:
-        command = get_command(tool_xml)
+        command = get_command(tool_xml) if tool_xml else None
         if command is None or not command.get("detect_errors"):
-            lint_ctx.info("No stdio definition found, tool will determine an error from stderr.")
+            if tool_source.parse_profile() <= "16.01":
+                lint_ctx.info("No stdio definition found, tool indicates error conditions with output written to stderr.")
+            else:
+                lint_ctx.info("No stdio definition found, tool indicates error conditions with non-zero exit codes.")
         return
 
     if len(stdios) > 1:

lib/galaxy/tools/linters/tests.py

@@ -21,10 +21,30 @@ def lint_tsts(tool_xml, lint_ctx):
             has_test = True
         if len(test.findall("assert_stdout")) > 0:
             has_test = True
-
-        outputs = test.findall("output") + test.findall("output_collection")
-        if len(outputs) > 0:
+        if len(test.findall("assert_command")) > 0:
             has_test = True
+
+        output_data_names, output_collection_names = _collect_output_names(tool_xml)
+        found_output_test = False
+        for output in test.findall("output"):
+            found_output_test = True
+            name = output.attrib.get("name", None)
+            if not name:
+                lint_ctx.warn("Found output tag without a name defined.")
+            else:
+                if name not in output_data_names:
+                    lint_ctx.warn("Found output tag with unknown name [%s], valid names [%s]" % (name, output_data_names))
+
+        for output_collection in test.findall("output_collection"):
+            found_output_test = True
+            name = output_collection.attrib.get("name", None)
+            if not name:
+                lint_ctx.warn("Found output_collection tag without a name defined.")
+            else:
+                if name not in output_collection_names:
+                    lint_ctx.warn("Found output_collection tag with unknown name [%s], valid names [%s]" % (name, output_collection_names))
+
+        has_test = has_test or found_output_test
         if not has_test:
             lint_ctx.warn("No outputs or expectations defined for tests, this test is likely invalid.")
         else:
@@ -34,3 +54,21 @@ def lint_tsts(tool_xml, lint_ctx):
         lint_ctx.valid("%d test(s) found.", num_valid_tests)
     else:
         lint_ctx.warn("No valid test(s) found.")
+
+
+def _collect_output_names(tool_xml):
+    output_data_names = []
+    output_collection_names = []
+
+    outputs = tool_xml.findall("./outputs")
+    if len(outputs) == 1:
+        for output in list(outputs[0]):
+            name = output.attrib.get("name", None)
+            if not name:
+                continue
+            if output.tag == "data":
+                output_data_names.append(name)
+            elif output.tag == "collection":
+                output_collection_names.append(name)
+
+    return output_data_names, output_collection_names

lib/galaxy/tools/toolbox/base.py

@@ -827,7 +827,7 @@ def reload_tool_by_id( self, tool_id ):
             # (Re-)Register the reloaded tool, this will handle
             #  _tools_by_id and _tool_versions_by_id
             self.register_tool( new_tool )
-            message = { 'name' : old_tool.name, 'id' : old_tool.id, 'version' : old_tool.version }
+            message = { 'name': old_tool.name, 'id': old_tool.id, 'version': old_tool.version }
             status = 'done'
         return message, status