Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rhpvorderman refactor shedtools #3

Merged
merged 8 commits into from
Aug 6, 2018
Merged

Rhpvorderman refactor shedtools #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7a6dd40
Fix extensions typo
mvdbeek Aug 5, 2018
b6ea9e1
Fix test_json default argument
mvdbeek Aug 5, 2018
7bc28cf
Only list repositories that are installed
mvdbeek Aug 5, 2018
57ac315
Listing data managers requires API access
mvdbeek Aug 5, 2018
16ede2a
Avoid `message` attr when passing exception msg
mvdbeek Aug 5, 2018
a43bcdb
Use named attributes of namedtuple
mvdbeek Aug 5, 2018
ed0ba61
Fix typo
mvdbeek Aug 5, 2018
cd46a4f
Avoid testing all installed tools
mvdbeek Aug 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 17 additions & 16 deletions ephemeris/get_tool_list_from_galaxy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,21 +115,22 @@ def record_repo(tool_elem):
# If someone knows a more effecient way around this problem it
# will be greatly appreciated.
for repo in repos:
for repo_with_panel in tools_with_panel:
tool_panel_section_id = None
tool_panel_section_label = None
if the_same_repository(repo_with_panel, repo, check_revision=False):
tool_panel_section_id = repo_with_panel.get('tool_panel_section_id')
tool_panel_section_label = repo_with_panel.get('tool_panel_section_label')
break
repositories.append(
dict(name=repo.get('name'),
owner=repo.get('owner'),
tool_shed_url=repo.get('tool_shed'),
revisions=[repo.get('changeset_revision')],
tool_panel_section_label=tool_panel_section_label,
tool_panel_section_id=tool_panel_section_id)
)
if not repo['deleted']:
for repo_with_panel in tools_with_panel:
tool_panel_section_id = None
tool_panel_section_label = None
if the_same_repository(repo_with_panel, repo, check_revision=False):
tool_panel_section_id = repo_with_panel.get('tool_panel_section_id')
tool_panel_section_label = repo_with_panel.get('tool_panel_section_label')
break
repositories.append(
dict(name=repo.get('name'),
owner=repo.get('owner'),
tool_shed_url=repo.get('tool_shed'),
revisions=[repo.get('changeset_revision')],
tool_panel_section_label=tool_panel_section_label,
tool_panel_section_id=tool_panel_section_id)
)
return repositories

@property
Expand Down Expand Up @@ -227,7 +228,7 @@ def _parser():
)
parser.add_argument("--get_data_managers",
action="store_true",
help="Include the data managers in the tool list. Requires login details")
help="Include the data managers in the tool list. Requires admin login details")
parser.add_argument("--get_all_tools",
action="store_true",
help="Get all tools and revisions, not just those which are present on the web ui."
Expand Down
46 changes: 21 additions & 25 deletions ephemeris/shed_tools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,20 +130,20 @@ def install_tools(self,
repository_list.append(complete_repo)
except (LookupError, KeyError) as e:
if log:
log_repository_install_error(repository, start, e.message, log)
log_repository_install_error(repository, start, str(e), log)
errored_repositories.append(repository)

# Filter out already installed repos
not_installed_repos, already_installed_repos = self.filter_installed_repos(repository_list)
filtered_repos = self.filter_installed_repos(repository_list)

for skipped_repo in already_installed_repos:
for skipped_repo in filtered_repos.already_installed_repos:
counter += 1
if log:
log_repository_install_skip(skipped_repo, counter, total_num_repositories, log)
skipped_repositories.append(skipped_repo)

# Install repos
for repository in not_installed_repos:
for repository in filtered_repos.not_installed_repos:
counter += 1
log_repository_install_start(repository, counter=counter, installation_start=installation_start, log=log,
total_num_repositories=total_num_repositories)
Expand Down Expand Up @@ -190,12 +190,12 @@ def update_tools(self, tools=None, log=None, **kwargs):
if not tools: # Tools None or empty list
tools = self.installed_tools()
else:
not_installed_tools, already_installed_tools = self.filter_installed_repos(tools, check_revision=False)
if not_installed_tools:
filtered_repos = self.filter_installed_repos(tools, check_revision=False)
if filtered_repos.not_installed_tools:
if log:
log.warning("The following tools are not installed and will not be upgraded: {0}".format(
not_installed_tools))
tools = already_installed_tools
filtered_repos.not_installed_tools))
tools = filtered_repos.already_installed_tools
return self.install_tools(tools, force_latest_revision=True, log=log, **kwargs)

def test_tools(self,
Expand Down Expand Up @@ -229,7 +229,7 @@ def test_tools(self,
results = self._test_tool(tool, test_user, test_user_api_key)
all_test_results.extend(results.tool_test_results)
tests_passed.extend(results.tests_passed)
test_exceptions.extend(results.test_extensions)
test_exceptions.extend(results.test_exceptions)

report_obj = {
'version': '0.1',
Expand Down Expand Up @@ -323,7 +323,7 @@ def install_repository_revision(self, repository, log):
log.debug("\tRepository %s already installed (at revision %s)" %
(repository['name'], repository['changeset_revision']))
return "skipped"
elif "504" in e.message:
elif "504" in str(e):
if log:
log.debug("Timeout during install of %s, extending wait to 1h", repository['name'])
success = self.wait_for_install(repository=repository, log=log, timeout=3600)
Expand Down Expand Up @@ -378,7 +378,7 @@ def wait_for_install(self, repository, log=None, timeout=3600):

def log_repository_install_error(repository, start, msg, log):
"""
Log failed tool installations. Return a dictionary wiyh information
Log failed tool installations. Return a dictionary with information
"""
end = dt.datetime.now()
log.error(
Expand Down Expand Up @@ -468,13 +468,9 @@ def main():

# Get some of the other installation arguments
kwargs = dict(
default_install_tool_dependencies=tool_list.get(
"install_tool_dependencies") or args.install_tool_dependencies,
default_install_repository_dependencies=tool_list.get(
"install_repository_dependencies") or args.install_repository_dependencies,
default_install_resolver_dependencies=tool_list.get(
"install_resolver_dependencies") or args.install_resolver_dependencies,
)
default_install_tool_dependencies=tool_list.get("install_tool_dependencies") or getattr(args, "install_tool_dependencies", False),
default_install_repository_dependencies=tool_list.get("install_repository_dependencies") or getattr(args, "install_repository_dependencies", False),
default_install_resolver_dependencies=tool_list.get("install_resolver_dependencies") or getattr(args, "install_resolver_dependencies", False))

# Start installing/updating and store the results in install_results.
# Or do testing if the action is `test`
Expand Down Expand Up @@ -505,13 +501,13 @@ def main():
to_be_tested_tools = install_results.installed_repositories
if args.test_existing:
to_be_tested_tools.extend(install_results.skipped_repositories)

install_tool_manager.test_tools(
test_json=args.test_json,
tools=to_be_tested_tools,
log=log,
test_user_api_key=args.test_user_api_key,
test_user=args.test_user)
if to_be_tested_tools:
install_tool_manager.test_tools(
test_json=args.test_json,
tools=to_be_tested_tools,
log=log,
test_user_api_key=args.test_user_api_key,
test_user=args.test_user)


if __name__ == "__main__":
Expand Down
2 changes: 2 additions & 0 deletions ephemeris/shed_tools_args.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ def parser():
command_parser.add_argument(
"--test_json",
dest="test_json",
default="tool_test_output.json",
help="If testing tools, record tool test output to specified file. "
"This file can be turned into reports with ``planemo test_reports <output.json>``."
)
Expand Down Expand Up @@ -194,6 +195,7 @@ def parser():
# Same test_json as above but language modified for test instead of install/update.
test_command_parser.add_argument(
"--test_json",
default="tool_test_output.json",
dest="test_json",
help="Record tool test output to specified file. "
"This file can be turned into reports with ``planemo test_reports <output.json>``."
Expand Down