Implement type-aware repository realization.

The biggest upshot here is that not now in source repositories package (tool_dependency_definition) and suite (repository_suite_definition) directories can now have readmes. The shed upload code will just quietly discard these.
galaxyproject · Apr 23, 2015 · 53edd99 · 53edd99
1 parent 1cf2f3e
commit 53edd99
Show file tree

Hide file tree

Showing 13 changed files with 779 additions and 18 deletions.
diff --git a/planemo/commands/cmd_shed_upload.py b/planemo/commands/cmd_shed_upload.py
@@ -82,7 +82,7 @@ def __handle_upload(ctx, path, **kwds):
     """
     tar_path = kwds.get("tar", None)
     if not tar_path:
-        tar_path = shed.build_tarball(path)
+        tar_path = shed.build_tarball(path, **kwds)
     if kwds["tar_only"]:
         shell("cp %s shed_upload.tar.gz" % tar_path)
         return 0

diff --git a/planemo/shed.py b/planemo/shed.py
@@ -53,6 +53,10 @@
                         " which is unavailable, please install `pip install "
                         "bioblend`")
 
+REPO_TYPE_UNRESTRICTED = "unrestricted"
+REPO_TYPE_TOOL_DEP = "tool_dependency_definition"
+REPO_TYPE_SUITE = "repository_suite_definition"
+
 
 def shed_repo_config(path):
     shed_yaml_path = os.path.join(path, SHED_CONFIG_NAME)
@@ -189,7 +193,7 @@ def download_tarball(ctx, tsi, path, **kwds):
                 os.remove(archival_file)
 
 
-def build_tarball(tool_path):
+def build_tarball(tool_path, **kwds):
     """Build a tool-shed tar ball for the specified path, caller is
     responsible for deleting this file.
     """
@@ -198,7 +202,8 @@ def build_tarball(tool_path):
     # It should be pushed up a level into the thing that is uploading tar
     # balls to iterate over them - but placing it here for now because
     # it address some bugs.
-    for realized_repository in realize_effective_repositories(tool_path):
+    effective_repositories = realize_effective_repositories(tool_path, **kwds)
+    for realized_repository in effective_repositories:
         fd, temp_path = mkstemp()
         try:
             tar = tarfile.open(temp_path, "w:gz")
@@ -252,11 +257,11 @@ def path_to_repo_name(path):
 def shed_repo_type(config, name):
     repo_type = config.get("type", None)
     if repo_type is None and name.startswith("package_"):
-        repo_type = "tool_dependency_definition"
+        repo_type = REPO_TYPE_TOOL_DEP
     elif repo_type is None and name.startswith("suite_"):
-        repo_type = "repository_suite_definition"
+        repo_type = REPO_TYPE_SUITE
     elif repo_type is None:
-        repo_type = "unrestricted"
+        repo_type = REPO_TYPE_UNRESTRICTED
     return repo_type
 
 
@@ -302,7 +307,7 @@ def _find_raw_repositories(path, **kwds):
         config = shed_repo_config(shed_file_dirs[0])
         config_name = config.get("name", None)
 
-    if len(shed_file_dirs) < 2 and config_name is None:
+    if len(shed_file_dirs) < 2 and config_name is None and name is None:
         name = path_to_repo_name(path)
 
     if len(shed_file_dirs) > 1 and name is not None:
@@ -348,13 +353,14 @@ def __init__(self, path, config):
         self.path = path
         self.config = config
         self.name = config["name"]
+        self.type = shed_repo_type(config, self.name)
 
     @property
     def _hash(self):
         return hashlib.md5(self.name.encode('utf-8')).hexdigest()
 
     def realize_to(self, parent_directory):
-        directory = os.path.join(parent_directory, self._hash)
+        directory = os.path.join(parent_directory, self._hash, self.name)
         if not os.path.exists(directory):
             os.makedirs(directory)
 
@@ -385,6 +391,16 @@ def _realize_file(self, relative_path, directory):
         os.symlink(source_path, target_path)
 
     def _implicit_ignores(self, relative_path):
+        # Filter out "unwanted files" :) like READMEs for special
+        # repository types.
+        if self.type == REPO_TYPE_TOOL_DEP:
+            if relative_path != "tool_dependencies.xml":
+                return True
+
+        if self.type == REPO_TYPE_SUITE:
+            if relative_path != "repository_dependencies.xml":
+                return True
+
         name = os.path.basename(relative_path)
         if relative_path.startswith(".git"):
             return True

diff --git a/planemo/shed_lint.py b/planemo/shed_lint.py
@@ -3,7 +3,12 @@
 import yaml
 from galaxy.tools.lint import LintContext
 from planemo.lint import lint_xsd
-from planemo.shed import path_to_repo_name
+from planemo.shed import (
+    path_to_repo_name,
+    REPO_TYPE_UNRESTRICTED,
+    REPO_TYPE_TOOL_DEP,
+    REPO_TYPE_SUITE,
+)
 from planemo.tool_lint import (
     build_lint_args,
     yield_tool_xmls,
@@ -24,9 +29,9 @@
 VALID_PUBLICNAME_RE = re.compile("^[a-z0-9\-]+$")
 
 VALID_REPOSITORY_TYPES = [
-    "unrestricted",
-    "tool_dependency_definition",
-    "repository_suite_definition",
+    REPO_TYPE_UNRESTRICTED,
+    REPO_TYPE_TOOL_DEP,
+    REPO_TYPE_SUITE,
 ]
 
 

diff --git a/tests/data/repos/package_1/.shed.yml b/tests/data/repos/package_1/.shed.yml
@@ -0,0 +1,3 @@
+name: package_1
+owner: iuc
+description: package_1 description
diff --git a/tests/data/repos/package_1/README.rst b/tests/data/repos/package_1/README.rst
@@ -0,0 +1,3 @@
+Test Package 1
+===============
+
diff --git a/tests/data/repos/package_1/tool_dependencies.xml b/tests/data/repos/package_1/tool_dependencies.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0"?>
+<tool_dependency>
+  <package name="samtools" version="0.1.16">
+    <install version="1.0">
+      <actions_group>
+        <actions os="linux" architecture="i386">
+          <action type="download_by_url" target_filename="samtools-0.1.16.tgz">http://depot.galaxyproject.org/package/linux/i386/samtools/samtools-0.1.16-linux-i386.tgz</action>
+          <action type="move_directory_files">
+            <source_directory>.</source_directory>
+            <destination_directory>$INSTALL_DIR</destination_directory>
+          </action>
+        </actions>
+        <actions os="linux" architecture="x86_64">
+          <action type="download_by_url" target_filename="samtools-0.1.16.tgz">http://depot.galaxyproject.org/package/linux/x86_64/samtools/samtools-0.1.16-linux-x86_64.tgz</action>
+          <action type="move_directory_files">
+            <source_directory>.</source_directory>
+            <destination_directory>$INSTALL_DIR</destination_directory>
+          </action>
+        </actions>
+        <actions os="darwin" architecture="i386">
+          <action type="download_by_url" target_filename="samtools-0.1.16.tgz">http://depot.galaxyproject.org/package/darwin/i386/samtools/samtools-0.1.16-Darwin-i386.tgz</action>
+          <action type="move_directory_files">
+            <source_directory>.</source_directory>
+            <destination_directory>$INSTALL_DIR</destination_directory>
+          </action>
+        </actions>
+        <actions os="darwin" architecture="x86_64">
+          <action type="download_by_url" target_filename="samtools-0.1.16.tgz">http://depot.galaxyproject.org/package/darwin/x86_64/samtools/samtools-0.1.16-Darwin-x86_64.tgz</action>
+          <action type="move_directory_files">
+            <source_directory>.</source_directory>
+            <destination_directory>$INSTALL_DIR</destination_directory>
+          </action>
+        </actions>
+        <actions>
+          <action type="download_by_url">http://depot.galaxyproject.org/package/source/samtools/samtools-0.1.16.tar.bz2</action>
+          <action type="shell_command">sed -i.bak 's/-lcurses/-lncurses/' Makefile</action>
+          <action type="shell_command">make</action>
+          <action type="move_file">
+            <source>samtools</source>
+            <destination>$INSTALL_DIR/bin</destination>
+          </action>
+          <action type="move_file">
+            <source>libbam.a</source>
+            <destination>$INSTALL_DIR/lib</destination>
+          </action>
+        </actions>
+        <action type="set_environment">
+          <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
+          <environment_variable name="BAM_LIB_PATH" action="set_to">$INSTALL_DIR/lib</environment_variable>
+        </action>
+      </actions_group>
+    </install>
+        <readme>
+          This is the last version of SAMTools to include the 'pileup' command.
+
+          Program: samtools (Tools for alignments in the SAM format)
+          Version: 0.1.16 (r963:234)
+
+          Usage:   samtools &lt;command&gt; [options]
+
+          Command: view        SAM&lt;-&gt;BAM conversion
+         sort        sort alignment file
+         pileup      generate pileup output
+         mpileup     multi-way pileup
+         depth       compute the depth
+         faidx       index/extract FASTA
+         tview       text alignment viewer
+         index       index alignment
+         idxstats    BAM index stats (r595 or later)
+         fixmate     fix mate information
+         glfview     print GLFv3 file
+         flagstat    simple stats
+         calmd       recalculate MD/NM tags and '=' bases
+         merge       merge sorted alignments
+         rmdup       remove PCR duplicates
+         reheader    replace BAM header
+         cat         concatenate BAMs
+         targetcut   cut fosmid regions (for fosmid pool only)
+         phase       phase heterozygotes
+        </readme>
+  </package>
+</tool_dependency>
diff --git a/tests/data/repos/suite_1/README.rst b/tests/data/repos/suite_1/README.rst
@@ -0,0 +1,2 @@
+Test Suite 1
+============
diff --git a/tests/data/repos/suite_1/repository_dependencies.xml b/tests/data/repos/suite_1/repository_dependencies.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<repositories description="A suite of Galaxy tools designed to work with version 1.2 of the SAMtools package.">
+    <repository name="data_manager_sam_fasta_index_builder" owner="devteam" />
+    <repository name="bam_to_sam" owner="devteam" />
+    <repository name="sam_to_bam" owner="devteam" />
+    <repository name="samtools_bedcov" owner="devteam" />
+    <repository name="samtools_calmd" owner="devteam" />
+    <repository name="samtools_flagstat" owner="devteam" />
+    <repository name="samtools_idxstat" owner="devteam" />
+    <repository name="samtools_mpileup" owner="devteam" />
+    <repository name="samtools_phase" owner="devteam" />
+    <repository name="samtools_reheader" owner="devteam" />
+    <repository name="samtools_rmdup" owner="devteam" />
+    <repository name="samtools_slice_bam" owner="devteam" />
+    <repository name="samtools_sort" owner="devteam" />
+    <repository name="samtools_split" owner="devteam" />
+    <repository name="samtools_stats" owner="devteam" />
+</repositories>
diff --git a/tests/data/repos/workflow_1/.shed.yml b/tests/data/repos/workflow_1/.shed.yml
@@ -0,0 +1,3 @@
+name: workflow_1
+owner: iuc
+description: workflow_1 description