Merge pull request #675 from jmchilton/mulled_register

Implement container_register for tool repositories.

planemo/commands/cmd_conda_install.py

@@ -11,6 +11,7 @@
 
 @click.command('conda_install')
 @options.optional_tools_or_packages_arg(multiple=True)
+@options.recursive_option()
 @options.conda_target_options()
 @options.conda_global_option()
 @options.conda_auto_init_option()
@@ -19,7 +20,7 @@ def cli(ctx, paths, **kwds):
     """Install conda packages for tool requirements."""
     conda_context = build_conda_context(ctx, handle_auto_init=True, **kwds)
     return_codes = []
-    for conda_target in collect_conda_targets(ctx, paths):
+    for conda_target in collect_conda_targets(ctx, paths, recursive=kwds["recursive"]):
         ctx.log("Install conda target %s" % conda_target)
         return_code = conda_util.install_conda_target(
             conda_target, conda_context=conda_context, skip_environment=kwds.get("global", False)

planemo/commands/cmd_container_register.py

@@ -0,0 +1,161 @@
+"""Module describing the planemo ``container_register`` command."""
+import os
+
+import click
+
+from galaxy.tools.deps.mulled.util import image_name, quay_repository
+
+from planemo import options
+from planemo.cli import command_function
+from planemo.conda import best_practice_search, collect_conda_target_lists
+from planemo.git import add, branch, commit, push
+from planemo.github_util import clone_fork_branch, get_repository_object, pull_request
+from planemo.mulled import conda_to_mulled_targets
+
+REGISTERY_TARGET_NAME = "multireqcontainers"
+REGISTERY_TARGET_PATH = "combinations"
+REGISTERY_REPOSITORY = "jmchilton/multireqcontainers"
+DEFAULT_MESSAGE = "Add %s (generated with Planemo)."
+
+
+@click.command('container_register')
+@options.optional_tools_arg(multiple=True)
+@options.recursive_option()
+@options.mulled_namespace_option()
+@click.option(
+    "output_directory",
+    "--output_directory",
+    type=click.Path(
+        file_okay=False,
+        dir_okay=True,
+        resolve_path=True,
+    ),
+    default=None,
+    help=("Container registration directory (defaults to ~/.planemo/multireqcontainers."),
+)
+@click.option(
+    "-m",
+    "--message",
+    default=DEFAULT_MESSAGE,
+    help="Commit and pull request message template for registration interactions."
+)
+@click.option(
+    "--pull_request/--no_pull_request",
+    is_flag=True,
+    default=True,
+    help="Fork and create a pull request against %s for these changes." % REGISTERY_REPOSITORY
+)
+@click.option(
+    "--force_push/--no_force_push",
+    is_flag=True,
+    default=False,
+    help="Force push branch for pull request in case it already exists.",
+)
+@command_function
+def cli(ctx, paths, **kwds):
+    """Register multi-requirement containers as needed.
+
+    BioContainers publishes all Bioconda packages automatically as individual
+    container images. These however are not enough for tools with multiple
+    best-practice requirements. Such requirements should be recorded and published
+    so that a container can be created and registered for these tools.
+    """
+    registry_target = RegistryTarget(ctx, **kwds)
+
+    combinations_added = 0
+    for conda_targets in collect_conda_target_lists(ctx, paths, recursive=kwds["recursive"]):
+        ctx.vlog("Handling conda_targets [%s]" % conda_targets)
+        mulled_targets = conda_to_mulled_targets(conda_targets)
+        if len(mulled_targets) < 2:
+            ctx.vlog("Skipping registeration, fewer than 2 targets discovered.")
+            # Skip these for now, we will want to revisit this for conda-forge dependencies and such.
+            continue
+
+        best_practice_requirements = True
+        for conda_target in conda_targets:
+            best_hit, exact = best_practice_search(conda_target)
+            if not best_hit or not exact:
+                ctx.vlog("Target [%s] is not available in best practice channels - skipping" % conda_target)
+                best_practice_requirements = False
+
+        if not best_practice_requirements:
+            continue
+
+        name = image_name(mulled_targets)
+        tag = "0"
+        name_and_tag = "%s:%s" % (name, tag)
+        target_filename = os.path.join(registry_target.output_directory, "%s.tsv" % name_and_tag)
+        ctx.vlog("Target filename for registeration is [%s]" % target_filename)
+        if os.path.exists(target_filename):
+            ctx.vlog("Target file already exists, skipping")
+            continue
+
+        namespace = kwds["mulled_namespace"]
+        repo_data = quay_repository(namespace, name)
+        if "tags" in repo_data:
+            ctx.vlog("quay repository already exists, skipping")
+            continue
+
+        if registry_target.has_pull_request_for(name):
+            ctx.vlog("Found matching open pull request for [%s], skipping" % name)
+            continue
+
+        registry_target.write_targets(ctx, target_filename, mulled_targets)
+        registry_target.handle_pull_request(ctx, name, target_filename, **kwds)
+        combinations_added += 1
+
+
+class RegistryTarget(object):
+    """Abstraction around mulled container registery (both directory and Github repo)."""
+
+    def __init__(self, ctx, **kwds):
+        output_directory = kwds["output_directory"]
+        pr_titles = []
+        target_repository = None
+        do_pull_request = kwds.get("pull_request", True)
+        if output_directory is None:
+            target_repository = os.path.join(ctx.workspace, REGISTERY_TARGET_NAME)
+            output_directory = os.path.join(target_repository, REGISTERY_TARGET_PATH)
+            clone_fork_branch(
+                ctx,
+                "https://github.com/%s" % REGISTERY_REPOSITORY,
+                target_repository,
+                fork=do_pull_request,
+            )
+            pr_titles = [pr.title for pr in open_prs(ctx)]
+
+        self.do_pull_request = do_pull_request
+        self.pr_titles = pr_titles
+        self.output_directory = output_directory
+        self.target_repository = target_repository
+
+    def has_pull_request_for(self, name):
+        has_pr = False
+        if self.do_pull_request:
+            if any([name in t for t in self.pr_titles]):
+                has_pr = True
+
+        return has_pr
+
+    def handle_pull_request(self, ctx, name, target_filename, **kwds):
+        if self.do_pull_request:
+            message = kwds["message"] % name
+            branch_name = name
+            branch(ctx, self.target_repository, branch_name, from_branch="master")
+            add(ctx, self.target_repository, target_filename)
+            commit(ctx, self.target_repository, message=message)
+            force_push = kwds.get("force_push", False)
+            push(ctx, self.target_repository, os.environ.get("GITHUB_USER"), branch_name, force=force_push)
+            pull_request(ctx, self.target_repository, message=message)
+
+    def write_targets(self, ctx, target_filename, mulled_targets):
+        with open(target_filename, "w") as f:
+            contents = ",".join(["%s=%s" % (t.package_name, t.version) for t in mulled_targets])
+            f.write(contents)
+            ctx.vlog("Wrote requirements [%s] to file [%s]" % (contents, target_filename))
+
+
+def open_prs(ctx):
+    repo = get_repository_object(ctx, REGISTERY_REPOSITORY)
+    prs = [pr for pr in repo.get_pulls()]
+    return prs

planemo/commands/cmd_mull.py

@@ -2,12 +2,10 @@
 import click
 
 from galaxy.tools.deps.mulled.mulled_build import mull_targets
-from galaxy.tools.deps.mulled.util import build_target
 
 from planemo import options
 from planemo.cli import command_function
-from planemo.conda import collect_conda_target_lists
-from planemo.mulled import build_mull_target_kwds
+from planemo.mulled import build_mull_target_kwds, collect_mulled_target_lists
 
 
 @click.command('mull')
@@ -29,8 +27,7 @@ def cli(ctx, paths, **kwds):
     This can be verified by running ``planemo lint --conda_requirements`` on the
     target tool(s).
     """
-    for conda_targets in collect_conda_target_lists(ctx, paths):
-        mulled_targets = map(lambda c: build_target(c.package, c.version), conda_targets)
+    for mulled_targets in collect_mulled_target_lists(ctx, paths, recursive=kwds["recursive"]):
         mull_target_kwds = build_mull_target_kwds(ctx, **kwds)
         command = kwds["mulled_command"]
         mull_targets(mulled_targets, command=command, **mull_target_kwds)

planemo/conda.py

@@ -6,6 +6,7 @@
 from __future__ import absolute_import
 
 import os
+import threading
 
 from galaxy.tools.deps import conda_util
 
@@ -17,6 +18,8 @@
 MESSAGE_ERROR_CANNOT_INSTALL = "Cannot install Conda - perhaps due to a failed installation or permission problems."
 MESSAGE_ERROR_NOT_INSTALLING = "Conda not configured - run ``planemo conda_init`` or pass ``--conda_auto_init`` to continue."
 
+BEST_PRACTICE_CHANNELS = ["conda-forge", "anaconda", "r", "bioconda"]
+
 
 def build_conda_context(ctx, **kwds):
     """Build a galaxy-lib CondaContext tailored to planemo use.
@@ -57,7 +60,7 @@ def build_conda_context(ctx, **kwds):
     return conda_context
 
 
-def collect_conda_targets(ctx, paths, found_tool_callback=None, conda_context=None):
+def collect_conda_targets(ctx, paths, recursive=False, found_tool_callback=None, conda_context=None):
     """Load CondaTarget objects from supplied artifact sources.
 
     If a tool contains more than one requirement, the requirements will each
@@ -72,7 +75,7 @@ def collect_conda_targets(ctx, paths, found_tool_callback=None, conda_context=No
         else:
             real_paths.append(path)
 
-    for (tool_path, tool_source) in yield_tool_sources_on_paths(ctx, real_paths):
+    for (tool_path, tool_source) in yield_tool_sources_on_paths(ctx, real_paths, recursive=recursive):
         if found_tool_callback:
             found_tool_callback(tool_path)
         for target in tool_source_conda_targets(tool_source):
@@ -95,14 +98,14 @@ def parse_target(target_str):
     return targets
 
 
-def collect_conda_target_lists(ctx, paths, found_tool_callback=None):
+def collect_conda_target_lists(ctx, paths, recursive=False, found_tool_callback=None):
     """Load CondaTarget lists from supplied artifact sources.
 
     If a tool contains more than one requirement, the requirements will all
     appear together as one list element of the output list.
     """
     conda_target_lists = set([])
-    for (tool_path, tool_source) in yield_tool_sources_on_paths(ctx, paths):
+    for (tool_path, tool_source) in yield_tool_sources_on_paths(ctx, paths, recursive=recursive, yield_load_errors=False):
         if found_tool_callback:
             found_tool_callback(tool_path)
         conda_target_lists.add(frozenset(tool_source_conda_targets(tool_source)))
@@ -115,7 +118,25 @@ def tool_source_conda_targets(tool_source):
     return conda_util.requirements_to_conda_targets(requirements)
 
 
+best_practice_search_first = threading.local()
+
+
+def best_practice_search(conda_target):
+    # Call it in offline mode after the first time.
+    try:
+        best_practice_search_first.previously_called
+        # TODO: Undo this...
+        offline = False
+    except AttributeError:
+        best_practice_search_first.previously_called = True
+        offline = False
+
+    return conda_util.best_search_result(conda_target, channels_override=BEST_PRACTICE_CHANNELS, offline=offline)
+
+
 __all__ = (
+    "BEST_PRACTICE_CHANNELS",
+    "best_practice_search",
     "build_conda_context",
     "collect_conda_targets",
     "collect_conda_target_lists",

planemo/git.py

@@ -16,6 +16,33 @@ def git_env_for(path):
     return env
 
 
+def add(ctx, repo_path, file_path):
+    env = git_env_for(repo_path)
+    io.communicate("cd '%s' && git add '%s'" % (repo_path, os.path.abspath(file_path)), env=env)
+
+
+def commit(ctx, repo_path, message=""):
+    env = git_env_for(repo_path)
+    io.communicate(["git", "commit", "-m", message], env=env)
+
+
+def push(ctx, repo_path, to, branch, force=False):
+    env = git_env_for(repo_path)
+    cmd = ["git", "push"]
+    if force:
+        cmd += ["--force"]
+    cmd += [to, branch]
+    io.communicate(cmd, env=env)
+
+
+def branch(ctx, repo_path, branch, from_branch=None):
+    env = git_env_for(repo_path)
+    cmd = ["git", "checkout", "-b", branch]
+    if from_branch is not None:
+        cmd.append(from_branch)
+    io.communicate(cmd, env=env)
+
+
 def checkout(ctx, remote_repo, local_path, branch=None, remote="origin", from_branch="master"):
     """Checkout a new branch from a remote repository."""
     env = git_env_for(local_path)

planemo/github_util.py

@@ -26,10 +26,10 @@
 FAILED_TO_DOWNLOAD_HUB = "No hub executable available and it could not be installed."
 
 
-def get_github_config(ctx):
+def get_github_config(ctx, allow_anonymous=False):
     """Return a :class:`planemo.github_util.GithubConfig` for given configuration."""
     global_github_config = _get_raw_github_config(ctx)
-    return None if global_github_config is None else GithubConfig(global_github_config)
+    return GithubConfig(global_github_config, allow_anonymous=allow_anonymous)
 
 
 def clone_fork_branch(ctx, target, path, **kwds):
@@ -43,7 +43,10 @@ def clone_fork_branch(ctx, target, path, **kwds):
         from_branch="master"
     )
     if kwds.get("fork"):
-        fork(ctx, path, **kwds)
+        try:
+            fork(ctx, path, **kwds)
+        except Exception:
+            pass
 
 
 def fork(ctx, path, **kwds):
@@ -108,7 +111,11 @@ def _try_download_hub(planemo_hub_path):
 def _get_raw_github_config(ctx):
     """Return a :class:`planemo.github_util.GithubConfig` for given configuration."""
     if "github" not in ctx.global_config:
-        return None
+        if "GITHUB_USER" in os.environ and "GITHUB_PASSWORD" in os.environ:
+            return {
+                "username": os.environ["GITHUB_USER"],
+                "password": os.environ["GITHUB_PASSWORD"],
+            }
     return ctx.global_config["github"]
 
 
@@ -118,10 +125,16 @@ class GithubConfig(object):
     Required to use ``github`` module methods that require authorization.
     """
 
-    def __init__(self, config):
+    def __init__(self, config, allow_anonymous=False):
         if not has_github_lib:
             raise Exception(NO_GITHUB_DEP_ERROR)
-        self._github = github.Github(config["username"], config["password"])
+        if "username" not in config or "password" not in config:
+            if not allow_anonymous:
+                raise Exception("github authentication unavailable")
+            github_object = github.Github()
+        else:
+            github_object = github.Github(config["username"], config["password"])
+        self._github = github_object
 
 
 def _hub_link():
@@ -137,14 +150,19 @@ def publish_as_gist_file(ctx, path, name="index"):
 
     More information on gists at http://gist.github.com/.
     """
-    github_config = get_github_config(ctx)
+    github_config = get_github_config(ctx, allow_anonymous=False)
     user = github_config._github.get_user()
     content = open(path, "r").read()
     content_file = github.InputFileContent(content)
     gist = user.create_gist(False, {name: content_file})
     return gist.files[name].raw_url
 
 
+def get_repository_object(ctx, name):
+    github_object = get_github_config(ctx, allow_anonymous=True)
+    return github_object._github.get_repo(name)
+
+
 __all__ = (
     "clone_fork_branch",
     "ensure_hub",