Skip to content

Commit 53edd99

Browse files
jmchiltonjmchilton
committed
Implement type-aware repository realization.
The biggest upshot here is that not now in source repositories package (tool_dependency_definition) and suite (repository_suite_definition) directories can now have readmes. The shed upload code will just quietly discard these.
1 parent 1cf2f3e commit 53edd99

File tree

13 files changed

+779
-18
lines changed

13 files changed

+779
-18
lines changed

‎planemo/commands/cmd_shed_upload.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ def __handle_upload(ctx, path, **kwds):
8282
"""
8383
tar_path = kwds.get("tar", None)
8484
if not tar_path:
85-
tar_path = shed.build_tarball(path)
85+
tar_path = shed.build_tarball(path, **kwds)
8686
if kwds["tar_only"]:
8787
shell("cp %s shed_upload.tar.gz" % tar_path)
8888
return 0

‎planemo/shed.py

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@
5353
" which is unavailable, please install `pip install "
5454
"bioblend`")
5555

56+
REPO_TYPE_UNRESTRICTED = "unrestricted"
57+
REPO_TYPE_TOOL_DEP = "tool_dependency_definition"
58+
REPO_TYPE_SUITE = "repository_suite_definition"
59+
5660

5761
def shed_repo_config(path):
5862
shed_yaml_path = os.path.join(path, SHED_CONFIG_NAME)
@@ -189,7 +193,7 @@ def download_tarball(ctx, tsi, path, **kwds):
189193
os.remove(archival_file)
190194

191195

192-
def build_tarball(tool_path):
196+
def build_tarball(tool_path, **kwds):
193197
"""Build a tool-shed tar ball for the specified path, caller is
194198
responsible for deleting this file.
195199
"""
@@ -198,7 +202,8 @@ def build_tarball(tool_path):
198202
# It should be pushed up a level into the thing that is uploading tar
199203
# balls to iterate over them - but placing it here for now because
200204
# it address some bugs.
201-
for realized_repository in realize_effective_repositories(tool_path):
205+
effective_repositories = realize_effective_repositories(tool_path, **kwds)
206+
for realized_repository in effective_repositories:
202207
fd, temp_path = mkstemp()
203208
try:
204209
tar = tarfile.open(temp_path, "w:gz")
@@ -252,11 +257,11 @@ def path_to_repo_name(path):
252257
def shed_repo_type(config, name):
253258
repo_type = config.get("type", None)
254259
if repo_type is None and name.startswith("package_"):
255-
repo_type = "tool_dependency_definition"
260+
repo_type = REPO_TYPE_TOOL_DEP
256261
elif repo_type is None and name.startswith("suite_"):
257-
repo_type = "repository_suite_definition"
262+
repo_type = REPO_TYPE_SUITE
258263
elif repo_type is None:
259-
repo_type = "unrestricted"
264+
repo_type = REPO_TYPE_UNRESTRICTED
260265
return repo_type
261266

262267

@@ -302,7 +307,7 @@ def _find_raw_repositories(path, **kwds):
302307
config = shed_repo_config(shed_file_dirs[0])
303308
config_name = config.get("name", None)
304309

305-
if len(shed_file_dirs) < 2 and config_name is None:
310+
if len(shed_file_dirs) < 2 and config_name is None and name is None:
306311
name = path_to_repo_name(path)
307312

308313
if len(shed_file_dirs) > 1 and name is not None:
@@ -348,13 +353,14 @@ def __init__(self, path, config):
348353
self.path = path
349354
self.config = config
350355
self.name = config["name"]
356+
self.type = shed_repo_type(config, self.name)
351357

352358
@property
353359
def _hash(self):
354360
return hashlib.md5(self.name.encode('utf-8')).hexdigest()
355361

356362
def realize_to(self, parent_directory):
357-
directory = os.path.join(parent_directory, self._hash)
363+
directory = os.path.join(parent_directory, self._hash, self.name)
358364
if not os.path.exists(directory):
359365
os.makedirs(directory)
360366

@@ -385,6 +391,16 @@ def _realize_file(self, relative_path, directory):
385391
os.symlink(source_path, target_path)
386392

387393
def _implicit_ignores(self, relative_path):
394+
# Filter out "unwanted files" :) like READMEs for special
395+
# repository types.
396+
if self.type == REPO_TYPE_TOOL_DEP:
397+
if relative_path != "tool_dependencies.xml":
398+
return True
399+
400+
if self.type == REPO_TYPE_SUITE:
401+
if relative_path != "repository_dependencies.xml":
402+
return True
403+
388404
name = os.path.basename(relative_path)
389405
if relative_path.startswith(".git"):
390406
return True

‎planemo/shed_lint.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,12 @@
33
import yaml
44
from galaxy.tools.lint import LintContext
55
from planemo.lint import lint_xsd
6-
from planemo.shed import path_to_repo_name
6+
from planemo.shed import (
7+
path_to_repo_name,
8+
REPO_TYPE_UNRESTRICTED,
9+
REPO_TYPE_TOOL_DEP,
10+
REPO_TYPE_SUITE,
11+
)
712
from planemo.tool_lint import (
813
build_lint_args,
914
yield_tool_xmls,
@@ -24,9 +29,9 @@
2429
VALID_PUBLICNAME_RE = re.compile("^[a-z0-9\-]+$")
2530

2631
VALID_REPOSITORY_TYPES = [
27-
"unrestricted",
28-
"tool_dependency_definition",
29-
"repository_suite_definition",
32+
REPO_TYPE_UNRESTRICTED,
33+
REPO_TYPE_TOOL_DEP,
34+
REPO_TYPE_SUITE,
3035
]
3136

3237

‎tests/data/repos/package_1/.shed.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
name: package_1
2+
owner: iuc
3+
description: package_1 description

‎tests/data/repos/package_1/README.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Test Package 1
2+
===============
3+

‎tests/data/repos/package_1/tool_dependencies.xml

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
<?xml version="1.0"?>
2+
<tool_dependency>
3+
<package name="samtools" version="0.1.16">
4+
<install version="1.0">
5+
<actions_group>
6+
<actions os="linux" architecture="i386">
7+
<action type="download_by_url" target_filename="samtools-0.1.16.tgz">http://depot.galaxyproject.org/package/linux/i386/samtools/samtools-0.1.16-linux-i386.tgz</action>
8+
<action type="move_directory_files">
9+
<source_directory>.</source_directory>
10+
<destination_directory>$INSTALL_DIR</destination_directory>
11+
</action>
12+
</actions>
13+
<actions os="linux" architecture="x86_64">
14+
<action type="download_by_url" target_filename="samtools-0.1.16.tgz">http://depot.galaxyproject.org/package/linux/x86_64/samtools/samtools-0.1.16-linux-x86_64.tgz</action>
15+
<action type="move_directory_files">
16+
<source_directory>.</source_directory>
17+
<destination_directory>$INSTALL_DIR</destination_directory>
18+
</action>
19+
</actions>
20+
<actions os="darwin" architecture="i386">
21+
<action type="download_by_url" target_filename="samtools-0.1.16.tgz">http://depot.galaxyproject.org/package/darwin/i386/samtools/samtools-0.1.16-Darwin-i386.tgz</action>
22+
<action type="move_directory_files">
23+
<source_directory>.</source_directory>
24+
<destination_directory>$INSTALL_DIR</destination_directory>
25+
</action>
26+
</actions>
27+
<actions os="darwin" architecture="x86_64">
28+
<action type="download_by_url" target_filename="samtools-0.1.16.tgz">http://depot.galaxyproject.org/package/darwin/x86_64/samtools/samtools-0.1.16-Darwin-x86_64.tgz</action>
29+
<action type="move_directory_files">
30+
<source_directory>.</source_directory>
31+
<destination_directory>$INSTALL_DIR</destination_directory>
32+
</action>
33+
</actions>
34+
<actions>
35+
<action type="download_by_url">http://depot.galaxyproject.org/package/source/samtools/samtools-0.1.16.tar.bz2</action>
36+
<action type="shell_command">sed -i.bak 's/-lcurses/-lncurses/' Makefile</action>
37+
<action type="shell_command">make</action>
38+
<action type="move_file">
39+
<source>samtools</source>
40+
<destination>$INSTALL_DIR/bin</destination>
41+
</action>
42+
<action type="move_file">
43+
<source>libbam.a</source>
44+
<destination>$INSTALL_DIR/lib</destination>
45+
</action>
46+
</actions>
47+
<action type="set_environment">
48+
<environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
49+
<environment_variable name="BAM_LIB_PATH" action="set_to">$INSTALL_DIR/lib</environment_variable>
50+
</action>
51+
</actions_group>
52+
</install>
53+
<readme>
54+
This is the last version of SAMTools to include the 'pileup' command.
55+
56+
Program: samtools (Tools for alignments in the SAM format)
57+
Version: 0.1.16 (r963:234)
58+
59+
Usage: samtools &lt;command&gt; [options]
60+
61+
Command: view SAM&lt;-&gt;BAM conversion
62+
sort sort alignment file
63+
pileup generate pileup output
64+
mpileup multi-way pileup
65+
depth compute the depth
66+
faidx index/extract FASTA
67+
tview text alignment viewer
68+
index index alignment
69+
idxstats BAM index stats (r595 or later)
70+
fixmate fix mate information
71+
glfview print GLFv3 file
72+
flagstat simple stats
73+
calmd recalculate MD/NM tags and '=' bases
74+
merge merge sorted alignments
75+
rmdup remove PCR duplicates
76+
reheader replace BAM header
77+
cat concatenate BAMs
78+
targetcut cut fosmid regions (for fosmid pool only)
79+
phase phase heterozygotes
80+
</readme>
81+
</package>
82+
</tool_dependency>

‎tests/data/repos/suite_1/README.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Test Suite 1
2+
============

‎tests/data/repos/suite_1/repository_dependencies.xml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<?xml version="1.0"?>
2+
<repositories description="A suite of Galaxy tools designed to work with version 1.2 of the SAMtools package.">
3+
<repository name="data_manager_sam_fasta_index_builder" owner="devteam" />
4+
<repository name="bam_to_sam" owner="devteam" />
5+
<repository name="sam_to_bam" owner="devteam" />
6+
<repository name="samtools_bedcov" owner="devteam" />
7+
<repository name="samtools_calmd" owner="devteam" />
8+
<repository name="samtools_flagstat" owner="devteam" />
9+
<repository name="samtools_idxstat" owner="devteam" />
10+
<repository name="samtools_mpileup" owner="devteam" />
11+
<repository name="samtools_phase" owner="devteam" />
12+
<repository name="samtools_reheader" owner="devteam" />
13+
<repository name="samtools_rmdup" owner="devteam" />
14+
<repository name="samtools_slice_bam" owner="devteam" />
15+
<repository name="samtools_sort" owner="devteam" />
16+
<repository name="samtools_split" owner="devteam" />
17+
<repository name="samtools_stats" owner="devteam" />
18+
</repositories>

‎tests/data/repos/workflow_1/.shed.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
name: workflow_1
2+
owner: iuc
3+
description: workflow_1 description

0 commit comments

Comments
 (0)