Permalink
Browse files

Update shed_diff and shed_download for multi-repo-shed-yamls.

 - Both realize repositories for configuration.
 - Both can now take in -r/--recursive arguments.
 - Download will attempt to download many repositories at once.
 - shed_diff now operates properly on the realized files instead of the local directory.

Still need to do a "smart diff" that filters out toolshed and changeset_revision attributes before #141 can be closed.
  • Loading branch information...
jmchilton committed May 1, 2015
1 parent 3a76f02 commit 40a1f579f7e326e9bd5dbe5312a5918cd8cb36d6
@@ -3,6 +3,7 @@
import os
import shutil
import tempfile
import sys

import click

@@ -30,6 +31,7 @@
" this to testtoolshed.",
default=None,
)
@options.recursive_shed_option()
@pass_context
def cli(ctx, path, **kwds):
"""Produce diff between local repository and Tool Shed contents.
@@ -50,14 +52,19 @@ def cli(ctx, path, **kwds):
% planemo shed_diff --owner peterjc --name blast_rbh
--shed_target_source testtoolshed
"""
working = tempfile.mkdtemp(prefix="tool_shed_diff_")
try:
diff_in(ctx, working, path, **kwds)
finally:
shutil.rmtree(working)
def diff(realized_repository):
working = tempfile.mkdtemp(prefix="tool_shed_diff_")
try:
diff_in(ctx, working, realized_repository, **kwds)
finally:
shutil.rmtree(working)

exit_code = shed.for_each_repository(diff, path, **kwds)
sys.exit(exit_code)

def diff_in(ctx, working, path, **kwds):

def diff_in(ctx, working, realized_repository, **kwds):
path = realized_repository.path
shed_target_source = kwds.get("shed_target_source", None)

label_a = "_%s_" % (shed_target_source if shed_target_source else "local")
@@ -73,20 +80,19 @@ def diff_in(ctx, working, path, **kwds):
shed.download_tarball(
ctx,
tsi,
path,
realized_repository,
destination=other,
clean=True,
**kwds
)

if shed_target_source:
new_kwds = kwds.copy()
new_kwds["shed_target"] = shed_target_source
tsi = shed.tool_shed_client(ctx, read_only=True, **new_kwds)
shed.download_tarball(
ctx,
tsi,
path,
realized_repository,
destination=mine,
clean=True,
**new_kwds
@@ -1,5 +1,7 @@
"""
"""
import sys

import click

from planemo.cli import pass_context
@@ -19,17 +21,26 @@
'--destination',
default="shed_download.tar.gz",
type=target_path,
help="Destination of tarball to download - if this doesn't end in 'gz' it "
"will be treated as a directory to extract tool contents into"
"(defaults to shed_download.tar.gz)."
help="Destination pattern of tarball(s) to download - if this doesn't "
"end in 'gz' it will be treated as a directory to extract tool "
"contents into (defaults to shed_download.tar.gz). If multiple "
"repositories are discovered in a .shed.yml file these will be "
"created as shed_download_<name>.tar.gz by default for instance, "
"simpler repositories will just be downloaded to the specified file."
)
@options.shed_owner_option()
@options.shed_name_option()
@options.shed_target_option()
@options.recursive_shed_option()
@pass_context
def cli(ctx, path, **kwds):
"""Download a tool repository as a tarball from the tool shed and extract
to the specified directory.
"""
tsi = shed.tool_shed_client(ctx, read_only=True, **kwds)
shed.download_tarball(ctx, tsi, path, **kwds)

def download(realized_repository):
return shed.download_tarball(ctx, tsi, realized_repository, **kwds)

exit_code = shed.for_each_repository(download, path, **kwds)
sys.exit(exit_code)
@@ -150,11 +150,8 @@ def upload_repository(ctx, realized_repository, **kwds):
if not tar_path:
tar_path = build_tarball(path, **kwds)
if kwds["tar_only"]:
suffix = ""
if realized_repository.multiple:
name = realized_repository.config["name"]
suffix = "_%s" % name.replace("-", "_")
shell("cp %s shed_upload%s.tar.gz" % (tar_path, suffix))
name = realized_repository.pattern_to_file_name("shed_upload.tar.gz")
shell("cp '%s' '%s'" % (tar_path, name))
return 0
tsi = tool_shed_client(ctx, **kwds)
update_kwds = {}
@@ -377,9 +374,10 @@ def create_repository_for(ctx, tsi, name, repo_config):
return repo


def download_tarball(ctx, tsi, path, **kwds):
repo_id = find_repository_id(ctx, tsi, path, **kwds)
destination = kwds.get('destination', 'shed_download.tar.gz')
def download_tarball(ctx, tsi, realized_repository, **kwds):
repo_id = realized_repository.find_repository_id(ctx, tsi)
destination_pattern = kwds.get('destination', 'shed_download.tar.gz')
destination = realized_repository.pattern_to_file_name(destination_pattern)
to_directory = not destination.endswith("gz")
download_tar(tsi, repo_id, destination, to_directory=to_directory)
if to_directory:
@@ -778,6 +776,19 @@ def __init__(self, realized_path, real_path, config, multiple):
self.name = config["name"]
self.multiple = multiple

def pattern_to_file_name(self, pattern):
if not self.multiple:
return pattern

name = self.config["name"]
suffix = "_%s" % name.replace("-", "_")

if "." not in pattern:
return pattern + suffix
else:
parts = pattern.split(".", 1)
return parts[0] + suffix + "." + parts[1]

def find_repository_id(self, ctx, tsi):
try:
repo_id = _find_repository_id(
@@ -0,0 +1,37 @@
from os.path import join

from .test_utils import CliShedTestCase
from .test_shed_upload import assert_exists


class ShedUploadTestCase(CliShedTestCase):

def test_download_expansion_tars(self):
with self._isolate_repo("multi_repos_flat_flag") as f:
upload_command = [
"shed_upload", "--force_repository_creation"
]
upload_command.extend(self._shed_args())
self._check_exit_code(upload_command)
download_command = [
"shed_download",
]
download_command.extend(self._shed_args(read_only=True))
self._check_exit_code(download_command)
assert_exists(join(f, "shed_download_cs_cat1.tar.gz"))
assert_exists(join(f, "shed_download_cs_cat2.tar.gz"))

def test_download_expansion_dir(self):
with self._isolate_repo("multi_repos_flat_flag") as f:
upload_command = [
"shed_upload", "--force_repository_creation"
]
upload_command.extend(self._shed_args())
self._check_exit_code(upload_command)
download_command = [
"shed_download", "--destination", "shed_d"
]
download_command.extend(self._shed_args(read_only=True))
self._check_exit_code(download_command)
assert_exists(join(f, "shed_d_cs_cat1", "cat1.xml"))
assert_exists(join(f, "shed_d_cs_cat2", "cat2.xml"))
@@ -16,7 +16,7 @@ def test_tar_single(self):
upload_command = ["shed_upload", "--tar_only"]
upload_command.extend(self._shed_args())
self._check_exit_code(upload_command)
_assert_exists(join(f, "shed_upload.tar.gz"))
assert_exists(join(f, "shed_upload.tar.gz"))

def test_upload_not_exists(self):
with self._isolate_repo("single_tool"):
@@ -68,7 +68,7 @@ def test_upload_filters_invalid_suite(self):
self._check_exit_code(upload_command)
target = self._untar(f, "shed_upload.tar.gz")
# Only one file was in archive
_assert_exists(join(target, "repository_dependencies.xml"))
assert_exists(join(target, "repository_dependencies.xml"))
# this got filtered
assert not exists(join(target, "README.rst"))

@@ -79,7 +79,7 @@ def test_upload_suite_auto(self):
self._check_exit_code(upload_command)
target = self._untar(f, "shed_upload_suite_1.tar.gz")
# Only one file was in archive
_assert_exists(join(target, "repository_dependencies.xml"))
assert_exists(join(target, "repository_dependencies.xml"))

def test_upload_filters_ignore(self):
with self._isolate_repo("single_tool_exclude") as f:
@@ -120,7 +120,7 @@ def test_upload_filters_invalid_package(self):
self._check_exit_code(upload_command)
target = self._untar(f, "shed_upload.tar.gz")
# Only one file was in archive
_assert_exists(join(target, "tool_dependencies.xml"))
assert_exists(join(target, "tool_dependencies.xml"))
# this got filtered
assert not exists(join(target, "README.rst"))
# .shed.yml always gets filtered
@@ -133,8 +133,8 @@ def test_upload_not_filters_unrestricted(self):
self._check_exit_code(upload_command)
target = self._untar(f, "shed_upload.tar.gz")
# Only one file was in archive
_assert_exists(join(target, "repository_dependencies.xml"))
_assert_exists(join(target, "README.rst"))
assert_exists(join(target, "repository_dependencies.xml"))
assert_exists(join(target, "README.rst"))

def test_upload_expansion_configured(self):
with self._isolate_repo("multi_repos_flat_configured") as f:
@@ -205,15 +205,15 @@ def _verify_single_uploaded(self, f):
def _verify_upload(self, f, download_files=[], download_args=[]):
target = self._download_repo(f, download_args)
for download_file in download_files:
_assert_exists(join(target, download_file))
assert_exists(join(target, download_file))
return target

def _check_tar(self, f, tar_path, contains=[], not_contains=[]):
tar_path = join(f, tar_path)
_assert_exists(tar_path)
assert_exists(tar_path)
target = self._untar(f, tar_path)
for path in contains:
_assert_exists(join(target, path))
assert_exists(join(target, path))
for path in not_contains:
assert not exists(join(target, path))
return target
@@ -224,7 +224,7 @@ def _download_repo(self, f, download_args=[]):
download_command.extend(self._shed_args(read_only=True))
self._check_exit_code(download_command)
download = join(f, "shed_download.tar.gz")
_assert_exists(download)
assert_exists(download)
return self._untar(f, "shed_download.tar.gz", tarbomb=False)

def _untar(self, f, path, tarbomb=True):
@@ -247,7 +247,7 @@ def _untar(self, f, path, tarbomb=True):
return target


def _assert_exists(path):
def assert_exists(path):
dir_path = os.path.dirname(path)
msg = None
if not exists(dir_path):

0 comments on commit 40a1f57

Please sign in to comment.