Browse files

Implement type-aware repository realization.

The biggest upshot here is that not now in source repositories package (tool_dependency_definition) and suite (repository_suite_definition) directories can now have readmes. The shed upload code will just quietly discard these.
  • Loading branch information...
jmchilton committed Apr 23, 2015
1 parent 1cf2f3e commit 53edd99a0859b77d5ac17e59016e23847562afd0
@@ -82,7 +82,7 @@ def __handle_upload(ctx, path, **kwds):
tar_path = kwds.get("tar", None)
if not tar_path:
tar_path = shed.build_tarball(path)
tar_path = shed.build_tarball(path, **kwds)
if kwds["tar_only"]:
shell("cp %s shed_upload.tar.gz" % tar_path)
return 0
@@ -53,6 +53,10 @@
" which is unavailable, please install `pip install "

REPO_TYPE_TOOL_DEP = "tool_dependency_definition"
REPO_TYPE_SUITE = "repository_suite_definition"

def shed_repo_config(path):
shed_yaml_path = os.path.join(path, SHED_CONFIG_NAME)
@@ -189,7 +193,7 @@ def download_tarball(ctx, tsi, path, **kwds):

def build_tarball(tool_path):
def build_tarball(tool_path, **kwds):
"""Build a tool-shed tar ball for the specified path, caller is
responsible for deleting this file.
@@ -198,7 +202,8 @@ def build_tarball(tool_path):
# It should be pushed up a level into the thing that is uploading tar
# balls to iterate over them - but placing it here for now because
# it address some bugs.
for realized_repository in realize_effective_repositories(tool_path):
effective_repositories = realize_effective_repositories(tool_path, **kwds)
for realized_repository in effective_repositories:
fd, temp_path = mkstemp()
tar =, "w:gz")
@@ -252,11 +257,11 @@ def path_to_repo_name(path):
def shed_repo_type(config, name):
repo_type = config.get("type", None)
if repo_type is None and name.startswith("package_"):
repo_type = "tool_dependency_definition"
repo_type = REPO_TYPE_TOOL_DEP
elif repo_type is None and name.startswith("suite_"):
repo_type = "repository_suite_definition"
repo_type = REPO_TYPE_SUITE
elif repo_type is None:
repo_type = "unrestricted"
return repo_type

@@ -302,7 +307,7 @@ def _find_raw_repositories(path, **kwds):
config = shed_repo_config(shed_file_dirs[0])
config_name = config.get("name", None)

if len(shed_file_dirs) < 2 and config_name is None:
if len(shed_file_dirs) < 2 and config_name is None and name is None:
name = path_to_repo_name(path)

if len(shed_file_dirs) > 1 and name is not None:
@@ -348,13 +353,14 @@ def __init__(self, path, config):
self.path = path
self.config = config = config["name"]
self.type = shed_repo_type(config,

def _hash(self):
return hashlib.md5('utf-8')).hexdigest()

def realize_to(self, parent_directory):
directory = os.path.join(parent_directory, self._hash)
directory = os.path.join(parent_directory, self._hash,
if not os.path.exists(directory):

@@ -385,6 +391,16 @@ def _realize_file(self, relative_path, directory):
os.symlink(source_path, target_path)

def _implicit_ignores(self, relative_path):
# Filter out "unwanted files" :) like READMEs for special
# repository types.
if self.type == REPO_TYPE_TOOL_DEP:
if relative_path != "tool_dependencies.xml":
return True

if self.type == REPO_TYPE_SUITE:
if relative_path != "repository_dependencies.xml":
return True

name = os.path.basename(relative_path)
if relative_path.startswith(".git"):
return True
@@ -3,7 +3,12 @@
import yaml
from import LintContext
from planemo.lint import lint_xsd
from planemo.shed import path_to_repo_name
from planemo.shed import (
from planemo.tool_lint import (
@@ -24,9 +29,9 @@
VALID_PUBLICNAME_RE = re.compile("^[a-z0-9\-]+$")


@@ -0,0 +1,3 @@
name: package_1
owner: iuc
description: package_1 description
@@ -0,0 +1,3 @@
Test Package 1

@@ -0,0 +1,82 @@
<?xml version="1.0"?>
<package name="samtools" version="0.1.16">
<install version="1.0">
<actions os="linux" architecture="i386">
<action type="download_by_url" target_filename="samtools-0.1.16.tgz"></action>
<action type="move_directory_files">
<actions os="linux" architecture="x86_64">
<action type="download_by_url" target_filename="samtools-0.1.16.tgz"></action>
<action type="move_directory_files">
<actions os="darwin" architecture="i386">
<action type="download_by_url" target_filename="samtools-0.1.16.tgz"></action>
<action type="move_directory_files">
<actions os="darwin" architecture="x86_64">
<action type="download_by_url" target_filename="samtools-0.1.16.tgz"></action>
<action type="move_directory_files">
<action type="download_by_url"></action>
<action type="shell_command">sed -i.bak 's/-lcurses/-lncurses/' Makefile</action>
<action type="shell_command">make</action>
<action type="move_file">
<action type="move_file">
<action type="set_environment">
<environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
<environment_variable name="BAM_LIB_PATH" action="set_to">$INSTALL_DIR/lib</environment_variable>
This is the last version of SAMTools to include the 'pileup' command.

Program: samtools (Tools for alignments in the SAM format)
Version: 0.1.16 (r963:234)

Usage: samtools &lt;command&gt; [options]

Command: view SAM&lt;-&gt;BAM conversion
sort sort alignment file
pileup generate pileup output
mpileup multi-way pileup
depth compute the depth
faidx index/extract FASTA
tview text alignment viewer
index index alignment
idxstats BAM index stats (r595 or later)
fixmate fix mate information
glfview print GLFv3 file
flagstat simple stats
calmd recalculate MD/NM tags and '=' bases
merge merge sorted alignments
rmdup remove PCR duplicates
reheader replace BAM header
cat concatenate BAMs
targetcut cut fosmid regions (for fosmid pool only)
phase phase heterozygotes
@@ -0,0 +1,2 @@
Test Suite 1
@@ -0,0 +1,18 @@
<?xml version="1.0"?>
<repositories description="A suite of Galaxy tools designed to work with version 1.2 of the SAMtools package.">
<repository name="data_manager_sam_fasta_index_builder" owner="devteam" />
<repository name="bam_to_sam" owner="devteam" />
<repository name="sam_to_bam" owner="devteam" />
<repository name="samtools_bedcov" owner="devteam" />
<repository name="samtools_calmd" owner="devteam" />
<repository name="samtools_flagstat" owner="devteam" />
<repository name="samtools_idxstat" owner="devteam" />
<repository name="samtools_mpileup" owner="devteam" />
<repository name="samtools_phase" owner="devteam" />
<repository name="samtools_reheader" owner="devteam" />
<repository name="samtools_rmdup" owner="devteam" />
<repository name="samtools_slice_bam" owner="devteam" />
<repository name="samtools_sort" owner="devteam" />
<repository name="samtools_split" owner="devteam" />
<repository name="samtools_stats" owner="devteam" />
@@ -0,0 +1,3 @@
name: workflow_1
owner: iuc
description: workflow_1 description
Oops, something went wrong.

0 comments on commit 53edd99

Please sign in to comment.