From dc1b335915c4fd5f2144c89d1a76e0c034677607 Mon Sep 17 00:00:00 2001 From: lorrainealisha75 Date: Tue, 23 Jun 2020 10:42:11 +0200 Subject: [PATCH 01/39] First Draft for autoupdate command --- planemo/commands/autoupdate/autoupdate.py | 60 +++++++++++++++++++ planemo/commands/autoupdate/cmd_autoupdate.py | 59 ++++++++++++++++++ planemo/commands/autoupdate/setup.py | 14 +++++ 3 files changed, 133 insertions(+) create mode 100644 planemo/commands/autoupdate/autoupdate.py create mode 100644 planemo/commands/autoupdate/cmd_autoupdate.py create mode 100644 planemo/commands/autoupdate/setup.py diff --git a/planemo/commands/autoupdate/autoupdate.py b/planemo/commands/autoupdate/autoupdate.py new file mode 100644 index 000000000..a63eb5fa0 --- /dev/null +++ b/planemo/commands/autoupdate/autoupdate.py @@ -0,0 +1,60 @@ +"""Autoupdate older conda dependencies in the requirements section.""" +from __future__ import absolute_import + +import planemo.conda + + +def autoupdate(tool_xml): + version_dict, main_req_dict = find_packages_in_requirements(tool_xml) + main_req_dict = get_main_requirement_version(tool_xml, main_req_dict) + updated_version_dict, main_req_updated_version = get_latest_versions(version_dict, main_req_dict) + update_requirements(tool_xml, updated_version_dict, main_req_updated_version) + + +def find_packages_in_requirements(tool_xml): + packages = {} + main_req_dict = {} + main_req = None + for element in tool_xml.getroot().findall("requirements"): + for requirement in list(element): + if requirement.tag == 'requirement' and requirement.attrib.get('type', '') == 'package': + if requirement.attrib.get('version', '') == '@TOOL_VERSION@': + main_req = requirement.text + packages[requirement.text] = requirement.attrib.get('version', '') + main_req_dict['main_req'] = main_req + return packages, main_req_dict + + +def get_main_requirement_version(tool_xml, main_req_dict): + for element in tool_xml.getroot().findall("macros"): + for macro in list(element): + if macro.tag == 'token' and macro.attrib.get('name', '') == '@TOOL_VERSION@': + main_req_dict['@TOOL_VERSION@'] = macro.text + return main_req_dict + return None + + +def get_latest_versions(version_dict, main_req_dict): + + for package in version_dict.keys(): + target = planemo.conda.conda_util.CondaTarget(package) + search_results = planemo.conda.best_practice_search(target) + version_dict[package] = search_results[0]['version'] + + main_req_dict['@TOOL_VERSION@'] = version_dict[main_req_dict['main_req']] + return version_dict, main_req_dict + + +def update_requirements(tool_xml, updated_version_dict, main_req_updated_version): + for element in tool_xml.getroot().findall("requirements"): + for requirement in list(element): + if requirement.tag == 'requirement' and requirement.attrib.get('type', '') == 'package': + if requirement.text == main_req_updated_version['main_req']: + requirement.set('version', '@TOOL_VERSION@') + else: + requirement.set('version', updated_version_dict[requirement.text]) + + for element in tool_xml.getroot().findall("macros"): + for macro in list(element): + if macro.tag == 'token' and macro.attrib.get('name', '') == '@TOOL_VERSION@': + macro.text = main_req_updated_version['@TOOL_VERSION@'] \ No newline at end of file diff --git a/planemo/commands/autoupdate/cmd_autoupdate.py b/planemo/commands/autoupdate/cmd_autoupdate.py new file mode 100644 index 000000000..c633e8474 --- /dev/null +++ b/planemo/commands/autoupdate/cmd_autoupdate.py @@ -0,0 +1,59 @@ +"""Module describing the planemo ``lint`` command.""" +import click + +# from os.path import basename + +# from galaxy.tool_util.lint import lint_tool_source + +from planemo.exit_codes import ( + EXIT_CODE_GENERIC_FAILURE, + EXIT_CODE_OK +) +from planemo.io import ( + coalesce_return_codes, + info +) +from planemo.tools import ( + is_tool_load_error, + yield_tool_sources_on_paths +) + +from planemo import options +from planemo.cli import command_function +from autoupdate import autoupdate + + +@click.command('autoupdate') +@options.optional_tools_arg(multiple=True, allow_uris=True) +@options.report_level_option() +@options.report_xunit() +@options.fail_level_option() +@options.skip_option() +@options.recursive_option() +@command_function +def cli(ctx, paths, **kwds): + """Check for common errors and best practices.""" + assert_tools = kwds.get("assert_tools", True) + recursive = kwds.get("recursive", False) + exit_codes = [] + for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): + info("Auto-updating tool %s" % tool_path) + autoupdate(tool_xml) + if handle_tool_load_error(tool_path, tool_xml): + exit_codes.append(EXIT_CODE_GENERIC_FAILURE) + continue + else: + exit_codes.append(EXIT_CODE_OK) + return coalesce_return_codes(exit_codes, assert_at_least_one=assert_tools) + ctx.exit() + + +def handle_tool_load_error(tool_path, tool_xml): + """ Return True if tool_xml is tool load error (invalid XML), and + print a helpful error message. + """ + is_error = False + if is_tool_load_error(tool_xml): + info("Could not update %s due to malformed xml." % tool_path) + is_error = True + return is_error diff --git a/planemo/commands/autoupdate/setup.py b/planemo/commands/autoupdate/setup.py new file mode 100644 index 000000000..8144a111d --- /dev/null +++ b/planemo/commands/autoupdate/setup.py @@ -0,0 +1,14 @@ +from setuptools import setup + +setup( + name='cmd_autoupdate', + version='0.1', + py_modules=['cmd_autoupdate'], + install_requires=[ + 'Click', + ], + entry_points=''' + [console_scripts] + cmd_autoupdate=cmd_autoupdate:cli + ''', +) \ No newline at end of file From c42db57660730fd346849f512a112b5cdae7339d Mon Sep 17 00:00:00 2001 From: lorrainealisha75 Date: Tue, 23 Jun 2020 11:23:45 +0200 Subject: [PATCH 02/39] Putting files in the right directories --- planemo/{commands/autoupdate => }/autoupdate.py | 0 planemo/commands/autoupdate/setup.py | 14 -------------- .../commands/{autoupdate => }/cmd_autoupdate.py | 0 3 files changed, 14 deletions(-) rename planemo/{commands/autoupdate => }/autoupdate.py (100%) delete mode 100644 planemo/commands/autoupdate/setup.py rename planemo/commands/{autoupdate => }/cmd_autoupdate.py (100%) diff --git a/planemo/commands/autoupdate/autoupdate.py b/planemo/autoupdate.py similarity index 100% rename from planemo/commands/autoupdate/autoupdate.py rename to planemo/autoupdate.py diff --git a/planemo/commands/autoupdate/setup.py b/planemo/commands/autoupdate/setup.py deleted file mode 100644 index 8144a111d..000000000 --- a/planemo/commands/autoupdate/setup.py +++ /dev/null @@ -1,14 +0,0 @@ -from setuptools import setup - -setup( - name='cmd_autoupdate', - version='0.1', - py_modules=['cmd_autoupdate'], - install_requires=[ - 'Click', - ], - entry_points=''' - [console_scripts] - cmd_autoupdate=cmd_autoupdate:cli - ''', -) \ No newline at end of file diff --git a/planemo/commands/autoupdate/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py similarity index 100% rename from planemo/commands/autoupdate/cmd_autoupdate.py rename to planemo/commands/cmd_autoupdate.py From dcf206a3e8232ed8f1a892dccfa9ada41b9c282e Mon Sep 17 00:00:00 2001 From: lorrainealisha75 Date: Wed, 24 Jun 2020 15:10:55 +0200 Subject: [PATCH 03/39] Making minor changes --- planemo/autoupdate.py | 4 +++- planemo/commands/cmd_autoupdate.py | 15 +++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index a63eb5fa0..bec9b2dde 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -15,6 +15,8 @@ def find_packages_in_requirements(tool_xml): packages = {} main_req_dict = {} main_req = None + print("fas") + print(tool_xml) for element in tool_xml.getroot().findall("requirements"): for requirement in list(element): if requirement.tag == 'requirement' and requirement.attrib.get('type', '') == 'package': @@ -57,4 +59,4 @@ def update_requirements(tool_xml, updated_version_dict, main_req_updated_version for element in tool_xml.getroot().findall("macros"): for macro in list(element): if macro.tag == 'token' and macro.attrib.get('name', '') == '@TOOL_VERSION@': - macro.text = main_req_updated_version['@TOOL_VERSION@'] \ No newline at end of file + macro.text = main_req_updated_version['@TOOL_VERSION@'] diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index c633e8474..364963441 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -1,10 +1,6 @@ -"""Module describing the planemo ``lint`` command.""" +"""Module describing the planemo ``autoupdate`` command.""" import click -# from os.path import basename - -# from galaxy.tool_util.lint import lint_tool_source - from planemo.exit_codes import ( EXIT_CODE_GENERIC_FAILURE, EXIT_CODE_OK @@ -18,13 +14,12 @@ yield_tool_sources_on_paths ) -from planemo import options +from planemo import options, autoupdate from planemo.cli import command_function -from autoupdate import autoupdate @click.command('autoupdate') -@options.optional_tools_arg(multiple=True, allow_uris=True) +@options.optional_tools_arg(multiple=True) @options.report_level_option() @options.report_xunit() @options.fail_level_option() @@ -32,13 +27,13 @@ @options.recursive_option() @command_function def cli(ctx, paths, **kwds): - """Check for common errors and best practices.""" + """Auto-update requirements section if necessary""" assert_tools = kwds.get("assert_tools", True) recursive = kwds.get("recursive", False) exit_codes = [] for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): info("Auto-updating tool %s" % tool_path) - autoupdate(tool_xml) + autoupdate.autoupdate(tool_xml) if handle_tool_load_error(tool_path, tool_xml): exit_codes.append(EXIT_CODE_GENERIC_FAILURE) continue From 81a8a83395b80121456a84ec859f29d73fd19a11 Mon Sep 17 00:00:00 2001 From: lorrainealisha75 Date: Thu, 25 Jun 2020 14:27:37 +0200 Subject: [PATCH 04/39] Basic functionalityof autoupdate check-in --- planemo/autoupdate.py | 51 ++++++++++-------------------- planemo/commands/cmd_autoupdate.py | 2 +- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index bec9b2dde..244583977 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -4,59 +4,40 @@ import planemo.conda -def autoupdate(tool_xml): +def begin_update(tool_path, tool_xml): version_dict, main_req_dict = find_packages_in_requirements(tool_xml) - main_req_dict = get_main_requirement_version(tool_xml, main_req_dict) - updated_version_dict, main_req_updated_version = get_latest_versions(version_dict, main_req_dict) - update_requirements(tool_xml, updated_version_dict, main_req_updated_version) + updated_version_dict = get_latest_versions(version_dict) + return update_requirements(tool_path, tool_xml, updated_version_dict) def find_packages_in_requirements(tool_xml): packages = {} main_req_dict = {} - main_req = None - print("fas") - print(tool_xml) - for element in tool_xml.getroot().findall("requirements"): + for element in tool_xml.xml_tree.findall("requirements"): for requirement in list(element): if requirement.tag == 'requirement' and requirement.attrib.get('type', '') == 'package': - if requirement.attrib.get('version', '') == '@TOOL_VERSION@': - main_req = requirement.text packages[requirement.text] = requirement.attrib.get('version', '') - main_req_dict['main_req'] = main_req return packages, main_req_dict -def get_main_requirement_version(tool_xml, main_req_dict): - for element in tool_xml.getroot().findall("macros"): - for macro in list(element): - if macro.tag == 'token' and macro.attrib.get('name', '') == '@TOOL_VERSION@': - main_req_dict['@TOOL_VERSION@'] = macro.text - return main_req_dict - return None - - -def get_latest_versions(version_dict, main_req_dict): +def get_latest_versions(version_dict): for package in version_dict.keys(): target = planemo.conda.conda_util.CondaTarget(package) search_results = planemo.conda.best_practice_search(target) version_dict[package] = search_results[0]['version'] - - main_req_dict['@TOOL_VERSION@'] = version_dict[main_req_dict['main_req']] - return version_dict, main_req_dict + return version_dict -def update_requirements(tool_xml, updated_version_dict, main_req_updated_version): - for element in tool_xml.getroot().findall("requirements"): +def update_requirements(tool_path, tool_xml, updated_version_dict): + for element in tool_xml.xml_tree.findall("requirements"): for requirement in list(element): if requirement.tag == 'requirement' and requirement.attrib.get('type', '') == 'package': - if requirement.text == main_req_updated_version['main_req']: - requirement.set('version', '@TOOL_VERSION@') - else: - requirement.set('version', updated_version_dict[requirement.text]) - - for element in tool_xml.getroot().findall("macros"): - for macro in list(element): - if macro.tag == 'token' and macro.attrib.get('name', '') == '@TOOL_VERSION@': - macro.text = main_req_updated_version['@TOOL_VERSION@'] + requirement.set('version', updated_version_dict[requirement.text]) + tool_xml.xml_tree.write(tool_path) + return tool_xml + + +__all__ = ( + "begin_update" +) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 364963441..c93bb847a 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -33,7 +33,7 @@ def cli(ctx, paths, **kwds): exit_codes = [] for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): info("Auto-updating tool %s" % tool_path) - autoupdate.autoupdate(tool_xml) + tool_xml = autoupdate.begin_update(tool_path, tool_xml) if handle_tool_load_error(tool_path, tool_xml): exit_codes.append(EXIT_CODE_GENERIC_FAILURE) continue From 85df40e1b926274fb0f86cc451162468ab450a31 Mon Sep 17 00:00:00 2001 From: lorrainealisha75 Date: Mon, 13 Jul 2020 11:09:54 +0200 Subject: [PATCH 05/39] Minor changens in autoupdate file --- planemo/autoupdate.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 244583977..c916582f1 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -5,19 +5,19 @@ def begin_update(tool_path, tool_xml): - version_dict, main_req_dict = find_packages_in_requirements(tool_xml) + version_dict = find_packages_in_requirements(tool_xml) updated_version_dict = get_latest_versions(version_dict) + macros = get_macros(tool_xml) return update_requirements(tool_path, tool_xml, updated_version_dict) def find_packages_in_requirements(tool_xml): packages = {} - main_req_dict = {} for element in tool_xml.xml_tree.findall("requirements"): for requirement in list(element): - if requirement.tag == 'requirement' and requirement.attrib.get('type', '') == 'package': - packages[requirement.text] = requirement.attrib.get('version', '') - return packages, main_req_dict + packages[requirement.text] = requirement.attrib.get('version') + #print(requirement) + return packages def get_latest_versions(version_dict): @@ -26,14 +26,20 @@ def get_latest_versions(version_dict): target = planemo.conda.conda_util.CondaTarget(package) search_results = planemo.conda.best_practice_search(target) version_dict[package] = search_results[0]['version'] + #print(version_dict) return version_dict +def get_macros(tool_xml): + for element in tool_xml.xml_tree.findall("macros"): + for macro in list(element): + print(macro.attrib) + + def update_requirements(tool_path, tool_xml, updated_version_dict): for element in tool_xml.xml_tree.findall("requirements"): for requirement in list(element): - if requirement.tag == 'requirement' and requirement.attrib.get('type', '') == 'package': - requirement.set('version', updated_version_dict[requirement.text]) + requirement.set('version', updated_version_dict[requirement.text]) tool_xml.xml_tree.write(tool_path) return tool_xml From 2b094184d485470f4319dafc2a8d1200e5c9beb4 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 29 Jul 2020 15:29:08 +0200 Subject: [PATCH 06/39] modify autoupdate so that tokens can be updated correctly --- planemo/autoupdate.py | 86 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 14 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index c916582f1..41ad7f13e 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -1,49 +1,107 @@ """Autoupdate older conda dependencies in the requirements section.""" from __future__ import absolute_import +import re +import xml.etree.ElementTree as ET import planemo.conda - def begin_update(tool_path, tool_xml): + tool_xml.xml_tree = ET.parse(tool_path) + macros = get_tokens(tool_xml) + version_dict = find_packages_in_requirements(tool_xml) - updated_version_dict = get_latest_versions(version_dict) - macros = get_macros(tool_xml) - return update_requirements(tool_path, tool_xml, updated_version_dict) + updated_main_req = get_latest_versions({version_dict['main_req']: tokens['@TOOL_VERSION@']}) + if updated_main_req[version_dict['main_req']] == tokens['@TOOL_VERSION@']: + return + + updated_version_dict = get_latest_versions(version_dict['other_reqs']) + return update_requirements(tool_path, tool_xml, updated_version_dict, updated_main_req) def find_packages_in_requirements(tool_xml): - packages = {} + """ + Get all requirements with versions as a dictionary of the form + {'main_req': main requirement, 'other_reqs': {req1: version, ... }} + """ + packages = {'other_reqs': {}} for element in tool_xml.xml_tree.findall("requirements"): for requirement in list(element): - packages[requirement.text] = requirement.attrib.get('version') - #print(requirement) + if requirement.attrib.get('version') == '@TOOL_VERSION@': + packages['main_req'] = requirement.text + else: + packages['other_reqs'][requirement.text] = requirement.attrib.get('version') return packages def get_latest_versions(version_dict): - + """ + Update a dict with current conda versions + """ for package in version_dict.keys(): target = planemo.conda.conda_util.CondaTarget(package) search_results = planemo.conda.best_practice_search(target) version_dict[package] = search_results[0]['version'] - #print(version_dict) return version_dict -def get_macros(tool_xml): +def get_tokens(tool_xml): + """ + Get tokens used to define versions + """ + macros = {} for element in tool_xml.xml_tree.findall("macros"): for macro in list(element): - print(macro.attrib) + if 'VERSION@' in macro.attrib['name']: + macros[macro.attrib['name']] = macro.text + return macros +def update_requirements(tool_path, tool_xml, updated_version_dict, updated_main_req): + """ + Update requirements to latest conda versions + and update version tokens + """ + for element in tool_xml.xml_tree.findall("macros"): + for token in list(element): + if token.attrib.get('name') == '@TOOL_VERSION@': + token.text = updated_main_req.values()[0] + elif token.attrib.get('name') == '@GALAXY_VERSION@': + token.text = '0' + else: + continue -def update_requirements(tool_path, tool_xml, updated_version_dict): for element in tool_xml.xml_tree.findall("requirements"): for requirement in list(element): - requirement.set('version', updated_version_dict[requirement.text]) - tool_xml.xml_tree.write(tool_path) + if requirement.text not in updated_main_req: + requirement.set('version', updated_version_dict[requirement.text]) + print(tool_path) + print(tool_xml) + + write_to_xml(tool_path, tool_xml) return tool_xml +def write_to_xml(tool_path, tool_xml): + """ + Write modified XML to tool_path + """ + # macros + m = tool_xml.xml_tree.find('macros') + m_str = ET.tostring(m).strip() + + # requirements + r = tool_xml.xml_tree.find('requirements') + r_str = ET.tostring(r).strip() + + # write to file + with open(tool_path, 'r+') as f: + xml_text = f.read() + xml_text = re.sub('(.|\n)*', m_str, xml_text) + xml_text = re.sub('(.|\n)*', r_str, xml_text) + f.seek(0) + f.truncate() + f.write(xml_text) + + __all__ = ( "begin_update" ) From 398828223f317aac072762a9449f0934da4537ae Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 30 Jul 2020 11:58:48 +0200 Subject: [PATCH 07/39] working version of autoupdate --- planemo/autoupdate.py | 143 +++++++++++++++-------------- planemo/commands/cmd_autoupdate.py | 2 +- 2 files changed, 75 insertions(+), 70 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 41ad7f13e..e4650bdd4 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -5,37 +5,57 @@ import xml.etree.ElementTree as ET import planemo.conda -def begin_update(tool_path, tool_xml): - tool_xml.xml_tree = ET.parse(tool_path) - macros = get_tokens(tool_xml) - - version_dict = find_packages_in_requirements(tool_xml) - updated_main_req = get_latest_versions({version_dict['main_req']: tokens['@TOOL_VERSION@']}) - if updated_main_req[version_dict['main_req']] == tokens['@TOOL_VERSION@']: - return - - updated_version_dict = get_latest_versions(version_dict['other_reqs']) - return update_requirements(tool_path, tool_xml, updated_version_dict, updated_main_req) - - -def find_packages_in_requirements(tool_xml): +def autoupdate(tool_path): + """ + Autoupdate an XML file + """ + xml_tree = ET.parse(tool_path) + requirements = find_requirements(xml_tree) + + for macro_import in requirements['imports']: + # recursively check macros + macro_requirements = autoupdate(macro_import) + for requirement in macro_requirements: + if requirement not in requirements: + requirements[requirement] = macro_requirements[requirement] + + # check main_req is up-to-date; if so, finish without changes + updated_main_req = get_latest_versions({requirements['main_req']: requirements['@TOOL_VERSION@']}) + if updated_main_req[requirements['main_req']] == requirements['@TOOL_VERSION@']: + return requirements + + # if main_req is not up-to-date, update everything + updated_version_dict = get_latest_versions(requirements['other_reqs']) + update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_req) + return requirements + +def find_requirements(xml_tree): """ Get all requirements with versions as a dictionary of the form - {'main_req': main requirement, 'other_reqs': {req1: version, ... }} + {'main_req': main requirement, 'other_reqs': {req1: version, ... }, + 'imports: ['macros.xml'], '*VERSION@': '...'} """ - packages = {'other_reqs': {}} - for element in tool_xml.xml_tree.findall("requirements"): - for requirement in list(element): - if requirement.attrib.get('version') == '@TOOL_VERSION@': - packages['main_req'] = requirement.text - else: - packages['other_reqs'][requirement.text] = requirement.attrib.get('version') - return packages + requirements = {'other_reqs': {}, 'imports': []} + + # get tokens + for token in xml_tree.iter("token"): + if 'VERSION@' in token.attrib.get('name', ''): + requirements[token.attrib['name']] = token.text + for macro_import in xml_tree.iter("import"): + requirements['imports'].append(macro_import.text) + + # get requirements + for requirement in xml_tree.iter("requirement"): + if requirement.attrib.get('version') == '@TOOL_VERSION@': + requirements['main_req'] = requirement.text + else: + requirements['other_reqs'][requirement.text] = requirement.attrib.get('version') + return requirements def get_latest_versions(version_dict): """ - Update a dict with current conda versions + Update a dict with current conda versions for tool requirements """ for package in version_dict.keys(): target = planemo.conda.conda_util.CondaTarget(package) @@ -44,64 +64,49 @@ def get_latest_versions(version_dict): return version_dict -def get_tokens(tool_xml): - """ - Get tokens used to define versions - """ - macros = {} - for element in tool_xml.xml_tree.findall("macros"): - for macro in list(element): - if 'VERSION@' in macro.attrib['name']: - macros[macro.attrib['name']] = macro.text - return macros - -def update_requirements(tool_path, tool_xml, updated_version_dict, updated_main_req): +def update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_req): """ Update requirements to latest conda versions and update version tokens """ - for element in tool_xml.xml_tree.findall("macros"): - for token in list(element): - if token.attrib.get('name') == '@TOOL_VERSION@': - token.text = updated_main_req.values()[0] - elif token.attrib.get('name') == '@GALAXY_VERSION@': - token.text = '0' - else: - continue - - for element in tool_xml.xml_tree.findall("requirements"): - for requirement in list(element): - if requirement.text not in updated_main_req: - requirement.set('version', updated_version_dict[requirement.text]) - print(tool_path) - print(tool_xml) - - write_to_xml(tool_path, tool_xml) - return tool_xml - - -def write_to_xml(tool_path, tool_xml): + + tags_to_update = {'tokens': [], 'requirements': []} + + for token in xml_tree.iter("token"): + if token.attrib.get('name') == '@TOOL_VERSION@': + token.text = updated_main_req.values()[0] + elif token.attrib.get('name') == '@GALAXY_VERSION@': + token.text = '0' + else: + continue + tags_to_update['tokens'].append(ET.tostring(token).strip()) + + for requirement in xml_tree.iter("requirement"): + if requirement.text not in updated_main_req: + requirement.set('version', updated_version_dict[requirement.text]) + tags_to_update['requirements'].append(ET.tostring(requirement).strip()) + write_to_xml(tool_path, xml_tree, tags_to_update) + return xml_tree + + +def write_to_xml(tool_path, xml_tree, tags_to_update): """ Write modified XML to tool_path """ - # macros - m = tool_xml.xml_tree.find('macros') - m_str = ET.tostring(m).strip() - - # requirements - r = tool_xml.xml_tree.find('requirements') - r_str = ET.tostring(r).strip() - - # write to file with open(tool_path, 'r+') as f: xml_text = f.read() - xml_text = re.sub('(.|\n)*', m_str, xml_text) - xml_text = re.sub('(.|\n)*', r_str, xml_text) + + for token in tags_to_update['tokens']: + xml_text = re.sub('{}>.*<{}'.format(*re.split('>.*<', token)), token, xml_text) + + for requirement in tags_to_update['requirements']: + xml_text = re.sub('{}version=".*"{}'.format(*re.split('version=".*"', requirement)), requirement, xml_text) + f.seek(0) f.truncate() f.write(xml_text) __all__ = ( - "begin_update" + "autoupdate" ) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index c93bb847a..32859c1c8 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -33,7 +33,7 @@ def cli(ctx, paths, **kwds): exit_codes = [] for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): info("Auto-updating tool %s" % tool_path) - tool_xml = autoupdate.begin_update(tool_path, tool_xml) + tool_xml = autoupdate.autoupdate(tool_path) if handle_tool_load_error(tool_path, tool_xml): exit_codes.append(EXIT_CODE_GENERIC_FAILURE) continue From efc78ee20893d710f0c7d85a46c8dcace231decc Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 3 Aug 2020 15:19:02 +0200 Subject: [PATCH 08/39] next commit --- planemo/autoupdate.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index e4650bdd4..81889152e 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -14,18 +14,20 @@ def autoupdate(tool_path): for macro_import in requirements['imports']: # recursively check macros - macro_requirements = autoupdate(macro_import) + macro_requirements = autoupdate('/'.join(tool_path.split('/')[:-1] + [macro_import])) for requirement in macro_requirements: if requirement not in requirements: requirements[requirement] = macro_requirements[requirement] + if not requirements.get('@TOOL_VERSION@'): + return requirements # check main_req is up-to-date; if so, finish without changes - updated_main_req = get_latest_versions({requirements['main_req']: requirements['@TOOL_VERSION@']}) - if updated_main_req[requirements['main_req']] == requirements['@TOOL_VERSION@']: + updated_main_req = get_latest_versions({requirements.get('main_req'): requirements.get('@TOOL_VERSION@')}) + if updated_main_req[requirements.get('main_req')] == requirements.get('@TOOL_VERSION@'): return requirements # if main_req is not up-to-date, update everything - updated_version_dict = get_latest_versions(requirements['other_reqs']) + updated_version_dict = get_latest_versions(requirements.get('other_reqs')) update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_req) return requirements From ce54d0f06fb538f6fca5f42faa520f70d24b11a6 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 6 Aug 2020 16:39:01 +0200 Subject: [PATCH 09/39] next commit --- planemo/autoupdate.py | 34 +++++++++++++++++++++++------- planemo/commands/cmd_autoupdate.py | 12 ++++++++++- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 81889152e..0366f4a79 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -5,7 +5,9 @@ import xml.etree.ElementTree as ET import planemo.conda -def autoupdate(tool_path): +from planemo.io import error, info + +def autoupdate(tool_path, dry_run=False): """ Autoupdate an XML file """ @@ -14,21 +16,29 @@ def autoupdate(tool_path): for macro_import in requirements['imports']: # recursively check macros - macro_requirements = autoupdate('/'.join(tool_path.split('/')[:-1] + [macro_import])) + macro_requirements = autoupdate('/'.join(tool_path.split('/')[:-1] + [macro_import]), dry_run) for requirement in macro_requirements: if requirement not in requirements: requirements[requirement] = macro_requirements[requirement] if not requirements.get('@TOOL_VERSION@'): + # if tool_version is not specified, finish without changes + error("The @TOOL_VERSION@ token is not specified. This is required for autoupdating.") return requirements - # check main_req is up-to-date; if so, finish without changes updated_main_req = get_latest_versions({requirements.get('main_req'): requirements.get('@TOOL_VERSION@')}) if updated_main_req[requirements.get('main_req')] == requirements.get('@TOOL_VERSION@'): + # check main_req is up-to-date; if so, finish without changes + info("No updates required or made.") return requirements + if dry_run: + error("Update required! Tool main requirement has version {}, newest conda version is {}".format(requirements.get('@TOOL_VERSION@'), updated_main_req[requirements.get('main_req')])) + return requirements + # if main_req is not up-to-date, update everything updated_version_dict = get_latest_versions(requirements.get('other_reqs')) update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_req) + info("Tool updated.") return requirements def find_requirements(xml_tree): @@ -76,17 +86,21 @@ def update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_ for token in xml_tree.iter("token"): if token.attrib.get('name') == '@TOOL_VERSION@': - token.text = updated_main_req.values()[0] + # check this + token.text = list(updated_main_req.values())[0] elif token.attrib.get('name') == '@GALAXY_VERSION@': token.text = '0' else: continue - tags_to_update['tokens'].append(ET.tostring(token).strip()) + tags_to_update['tokens'].append(ET.tostring(token, encoding="unicode").strip()) + + if '@GALAXY_VERSION@' not in [n.attrib.get('name') for n in xml_tree.iter('token')]: + tags_to_update['update_tool'] = True for requirement in xml_tree.iter("requirement"): if requirement.text not in updated_main_req: requirement.set('version', updated_version_dict[requirement.text]) - tags_to_update['requirements'].append(ET.tostring(requirement).strip()) + tags_to_update['requirements'].append(ET.tostring(requirement, encoding="unicode").strip()) write_to_xml(tool_path, xml_tree, tags_to_update) return xml_tree @@ -97,18 +111,22 @@ def write_to_xml(tool_path, xml_tree, tags_to_update): """ with open(tool_path, 'r+') as f: xml_text = f.read() - + print(tags_to_update) for token in tags_to_update['tokens']: xml_text = re.sub('{}>.*<{}'.format(*re.split('>.*<', token)), token, xml_text) for requirement in tags_to_update['requirements']: xml_text = re.sub('{}version=".*"{}'.format(*re.split('version=".*"', requirement)), requirement, xml_text) + # if '@GALAXY_VERSION@' not in tags_to_update['tokens']: + if tags_to_update.get('update_tool'): + # update the version directly in the tool tag + xml_text = re.sub('version="@TOOL_VERSION@\+galaxy.*"', 'version="@TOOL_VERSION@+galaxy0"', xml_text) + f.seek(0) f.truncate() f.write(xml_text) - __all__ = ( "autoupdate" ) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 32859c1c8..5d26db2d8 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -16,8 +16,17 @@ from planemo import options, autoupdate from planemo.cli import command_function +from planemo.config import planemo_option +def dry_run_option(): + """Perform a dry run autoupdate without modifying the XML files""" + return planemo_option( + "--dry-run", + is_flag=True, + help="Perform a dry run autoupdate without modifying the XML files." + ) + @click.command('autoupdate') @options.optional_tools_arg(multiple=True) @options.report_level_option() @@ -25,6 +34,7 @@ @options.fail_level_option() @options.skip_option() @options.recursive_option() +@dry_run_option() @command_function def cli(ctx, paths, **kwds): """Auto-update requirements section if necessary""" @@ -33,7 +43,7 @@ def cli(ctx, paths, **kwds): exit_codes = [] for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): info("Auto-updating tool %s" % tool_path) - tool_xml = autoupdate.autoupdate(tool_path) + tool_xml = autoupdate.autoupdate(tool_path, kwds['dry_run']) if handle_tool_load_error(tool_path, tool_xml): exit_codes.append(EXIT_CODE_GENERIC_FAILURE) continue From f7dc089513f09883a6a0af8eb6debc9cc8129b30 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Fri, 7 Aug 2020 10:39:37 +0200 Subject: [PATCH 10/39] minor change to logging --- planemo/autoupdate.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 0366f4a79..16f1cdcb1 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -23,22 +23,22 @@ def autoupdate(tool_path, dry_run=False): if not requirements.get('@TOOL_VERSION@'): # if tool_version is not specified, finish without changes - error("The @TOOL_VERSION@ token is not specified. This is required for autoupdating.") + error("The @TOOL_VERSION@ token is not specified in {}. This is required for autoupdating.".format(tool_path)) return requirements updated_main_req = get_latest_versions({requirements.get('main_req'): requirements.get('@TOOL_VERSION@')}) if updated_main_req[requirements.get('main_req')] == requirements.get('@TOOL_VERSION@'): # check main_req is up-to-date; if so, finish without changes - info("No updates required or made.") + info("No updates required or madeto {}.".format(tool_path)) return requirements if dry_run: - error("Update required! Tool main requirement has version {}, newest conda version is {}".format(requirements.get('@TOOL_VERSION@'), updated_main_req[requirements.get('main_req')])) + error("Update required to {}! Tool main requirement has version {}, newest conda version is {}".format(tool_path, requirements.get('@TOOL_VERSION@'), updated_main_req[requirements.get('main_req')])) return requirements - + # if main_req is not up-to-date, update everything updated_version_dict = get_latest_versions(requirements.get('other_reqs')) update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_req) - info("Tool updated.") + info("Tool {} updated.".format(tool_path)) return requirements def find_requirements(xml_tree): @@ -111,7 +111,6 @@ def write_to_xml(tool_path, xml_tree, tags_to_update): """ with open(tool_path, 'r+') as f: xml_text = f.read() - print(tags_to_update) for token in tags_to_update['tokens']: xml_text = re.sub('{}>.*<{}'.format(*re.split('>.*<', token)), token, xml_text) From cd139a6dbcdb2e29a6797bb52d23e18e7a203e89 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 13 Aug 2020 14:12:34 +0200 Subject: [PATCH 11/39] add conda flags --- planemo/autoupdate.py | 23 +++++++++++++++-------- planemo/commands/cmd_autoupdate.py | 3 ++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 16f1cdcb1..2bf2390d2 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -5,9 +5,10 @@ import xml.etree.ElementTree as ET import planemo.conda +from galaxy.tool_util.deps import conda_util from planemo.io import error, info -def autoupdate(tool_path, dry_run=False): +def autoupdate(ctx, tool_path, **kwds): """ Autoupdate an XML file """ @@ -25,22 +26,23 @@ def autoupdate(tool_path, dry_run=False): # if tool_version is not specified, finish without changes error("The @TOOL_VERSION@ token is not specified in {}. This is required for autoupdating.".format(tool_path)) return requirements - updated_main_req = get_latest_versions({requirements.get('main_req'): requirements.get('@TOOL_VERSION@')}) + updated_main_req = get_latest_versions(ctx, {requirements.get('main_req'): requirements.get('@TOOL_VERSION@')}, **kwds) if updated_main_req[requirements.get('main_req')] == requirements.get('@TOOL_VERSION@'): # check main_req is up-to-date; if so, finish without changes - info("No updates required or madeto {}.".format(tool_path)) + info("No updates required or made to {}.".format(tool_path)) return requirements - if dry_run: + if kwds.get('dry_run'): error("Update required to {}! Tool main requirement has version {}, newest conda version is {}".format(tool_path, requirements.get('@TOOL_VERSION@'), updated_main_req[requirements.get('main_req')])) return requirements # if main_req is not up-to-date, update everything - updated_version_dict = get_latest_versions(requirements.get('other_reqs')) + updated_version_dict = get_latest_versions(ctx, requirements.get('other_reqs'), **kwds) update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_req) info("Tool {} updated.".format(tool_path)) return requirements + def find_requirements(xml_tree): """ Get all requirements with versions as a dictionary of the form @@ -65,14 +67,19 @@ def find_requirements(xml_tree): return requirements -def get_latest_versions(version_dict): +def get_latest_versions(ctx, version_dict, **kwds): """ Update a dict with current conda versions for tool requirements """ for package in version_dict.keys(): + conda_context = planemo.conda.build_conda_context(ctx, **kwds) target = planemo.conda.conda_util.CondaTarget(package) - search_results = planemo.conda.best_practice_search(target) - version_dict[package] = search_results[0]['version'] + search_results = conda_util.best_search_result(target, conda_context=conda_context) + if search_results[1]: + version_dict[package] = search_results[0]['version'] + else: + error('Requirements could not be found!') + exit() return version_dict diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 5d26db2d8..1d00ae9bb 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -29,6 +29,7 @@ def dry_run_option(): @click.command('autoupdate') @options.optional_tools_arg(multiple=True) +@options.conda_target_options() @options.report_level_option() @options.report_xunit() @options.fail_level_option() @@ -43,7 +44,7 @@ def cli(ctx, paths, **kwds): exit_codes = [] for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): info("Auto-updating tool %s" % tool_path) - tool_xml = autoupdate.autoupdate(tool_path, kwds['dry_run']) + tool_xml = autoupdate.autoupdate(ctx, tool_path, **kwds) if handle_tool_load_error(tool_path, tool_xml): exit_codes.append(EXIT_CODE_GENERIC_FAILURE) continue From b572feeef346cc3e7cda05e9b26a3de99bbe99ab Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 13 Aug 2020 15:11:12 +0200 Subject: [PATCH 12/39] linting --- planemo/autoupdate.py | 12 ++++++++---- planemo/commands/cmd_autoupdate.py | 8 ++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 2bf2390d2..a1c1c85e1 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -3,11 +3,13 @@ import re import xml.etree.ElementTree as ET -import planemo.conda from galaxy.tool_util.deps import conda_util + +import planemo.conda from planemo.io import error, info + def autoupdate(ctx, tool_path, **kwds): """ Autoupdate an XML file @@ -17,7 +19,7 @@ def autoupdate(ctx, tool_path, **kwds): for macro_import in requirements['imports']: # recursively check macros - macro_requirements = autoupdate('/'.join(tool_path.split('/')[:-1] + [macro_import]), dry_run) + macro_requirements = autoupdate('/'.join(tool_path.split('/')[:-1] + [macro_import]), kwds.get('dry_run')) for requirement in macro_requirements: if requirement not in requirements: requirements[requirement] = macro_requirements[requirement] @@ -33,7 +35,8 @@ def autoupdate(ctx, tool_path, **kwds): return requirements if kwds.get('dry_run'): - error("Update required to {}! Tool main requirement has version {}, newest conda version is {}".format(tool_path, requirements.get('@TOOL_VERSION@'), updated_main_req[requirements.get('main_req')])) + error("Update required to {}! Tool main requirement has version {}, newest conda version is {}".format( + tool_path, requirements.get('@TOOL_VERSION@'), updated_main_req[requirements.get('main_req')])) return requirements # if main_req is not up-to-date, update everything @@ -127,12 +130,13 @@ def write_to_xml(tool_path, xml_tree, tags_to_update): # if '@GALAXY_VERSION@' not in tags_to_update['tokens']: if tags_to_update.get('update_tool'): # update the version directly in the tool tag - xml_text = re.sub('version="@TOOL_VERSION@\+galaxy.*"', 'version="@TOOL_VERSION@+galaxy0"', xml_text) + xml_text = re.sub(r'version="@TOOL_VERSION@\+galaxy.*"', 'version="@TOOL_VERSION@+galaxy0"', xml_text) f.seek(0) f.truncate() f.write(xml_text) + __all__ = ( "autoupdate" ) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 1d00ae9bb..5975934ec 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -1,6 +1,9 @@ """Module describing the planemo ``autoupdate`` command.""" import click +from planemo import autoupdate, options +from planemo.cli import command_function +from planemo.config import planemo_option from planemo.exit_codes import ( EXIT_CODE_GENERIC_FAILURE, EXIT_CODE_OK @@ -14,10 +17,6 @@ yield_tool_sources_on_paths ) -from planemo import options, autoupdate -from planemo.cli import command_function -from planemo.config import planemo_option - def dry_run_option(): """Perform a dry run autoupdate without modifying the XML files""" @@ -27,6 +26,7 @@ def dry_run_option(): help="Perform a dry run autoupdate without modifying the XML files." ) + @click.command('autoupdate') @options.optional_tools_arg(multiple=True) @options.conda_target_options() From f3160e351bf75ede50198112241ea0f5a5c1f667 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 13 Aug 2020 15:37:26 +0200 Subject: [PATCH 13/39] small changes to docs --- planemo/autoupdate.py | 2 +- planemo/commands/cmd_autoupdate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index a1c1c85e1..523d20588 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -50,7 +50,7 @@ def find_requirements(xml_tree): """ Get all requirements with versions as a dictionary of the form {'main_req': main requirement, 'other_reqs': {req1: version, ... }, - 'imports: ['macros.xml'], '*VERSION@': '...'} + 'imports: ['macros.xml'], '*VERSION@': '...'} """ requirements = {'other_reqs': {}, 'imports': []} diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 5975934ec..9bdd43501 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -38,7 +38,7 @@ def dry_run_option(): @dry_run_option() @command_function def cli(ctx, paths, **kwds): - """Auto-update requirements section if necessary""" + """Auto-update tool requirements by checking against Conda and updating if newer versions are available""" assert_tools = kwds.get("assert_tools", True) recursive = kwds.get("recursive", False) exit_codes = [] From b52825db94e94c6cb1e7772c60fa9decb70cc413 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 13 Aug 2020 17:46:16 +0200 Subject: [PATCH 14/39] add initial draft of some documentation --- docs/autoupdate.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/autoupdate.rst diff --git a/docs/autoupdate.rst b/docs/autoupdate.rst new file mode 100644 index 000000000..38e0e6fdc --- /dev/null +++ b/docs/autoupdate.rst @@ -0,0 +1,36 @@ +.. _shed: + +============================= +Autoupdating tools +============================= + +Galaxy tools use underlying command-line software which are specified using ```` tags in the XML wrapper. If developers continue to release new versions of the software, the creator/maintainer needs to ensure that the tool requirements are bumped, otherwise the tool will become updated. + +Planemo provides a ``autoupdate`` subcommand which can be used to perform this task automatically. Basic usage is as follows: + +:: + + planemo autoupdate tool_wrapper.xml + +There are various flags which can be applied; two of the most important are ``--recursive``, which performs the autoupdate recursively on subdirectories, and ``--dry-run``, which checks if tool requirements are up-to-date without making the necessary change automatically. + +One of the most efficient ways to use it is by implementing a CI cron job which runs the command on an entire GitHub repository of tools. + +Formatting tools +============================= + + ``autoupdate`` assumes that XML tools are formatted in a certain way - in accordance with the `IUC best practices`_ + +`__. In particular: + + - the tool ``version`` attribute must be set to ``@TOOL_VERSION@+galaxy0`` or ``@TOOL_VERSION@+galaxy@GALAXY_VERSION`` + - A token ``@TOOL_VERSION@`` should be created which corresponds to the version number of the main requirement. + - Optionally, a token ``@GALAXY_VERSION@`` should be created, which should be an integer representing the number of times the XML wrapper has been updated since ``@TOOL_VERSION@`` was updated. + + +Implementing an autoupdate CI job +============================= + +WIP + +.. _IUC best practices: https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices/tool_xml.html From 3ba07754760d066ecf28a128b5f2bd924911577d Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Fri, 14 Aug 2020 14:45:18 +0200 Subject: [PATCH 15/39] some fixes --- planemo/autoupdate.py | 31 ++++++++++++++++++++++++------ planemo/commands/cmd_autoupdate.py | 9 +++++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 523d20588..0bf7228da 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -16,14 +16,12 @@ def autoupdate(ctx, tool_path, **kwds): """ xml_tree = ET.parse(tool_path) requirements = find_requirements(xml_tree) - for macro_import in requirements['imports']: # recursively check macros - macro_requirements = autoupdate('/'.join(tool_path.split('/')[:-1] + [macro_import]), kwds.get('dry_run')) + macro_requirements = autoupdate(ctx, '/'.join(tool_path.split('/')[:-1] + [macro_import]), **kwds) #kwds.get('dry_run')) for requirement in macro_requirements: if requirement not in requirements: requirements[requirement] = macro_requirements[requirement] - if not requirements.get('@TOOL_VERSION@'): # if tool_version is not specified, finish without changes error("The @TOOL_VERSION@ token is not specified in {}. This is required for autoupdating.".format(tool_path)) @@ -94,23 +92,43 @@ def update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_ tags_to_update = {'tokens': [], 'requirements': []} + # get name of token which defines the wrapper version; if just an integer, None + versions = xml_tree.getroot().attrib.get('version')#.split('+galaxy') + if versions: + versions = versions.split('+galaxy') + if len(versions) == 1: + error('Tool version is not specified as two parts separated by +galaxy as required by autoupdate.') + exit() + elif versions[0] != '@TOOL_VERSION@': + error('Tool version does not contain @TOOL_VERSION@ as required by autoupdate.') + exit() + else: + if versions[1][0] == versions[1][-1] == '@': + wrapper_token = versions[1] + else: + wrapper_token = None + else: + wrapper_token = None + print(wrapper_token) + for token in xml_tree.iter("token"): if token.attrib.get('name') == '@TOOL_VERSION@': # check this token.text = list(updated_main_req.values())[0] - elif token.attrib.get('name') == '@GALAXY_VERSION@': + elif token.attrib.get('name') == wrapper_token: token.text = '0' else: continue tags_to_update['tokens'].append(ET.tostring(token, encoding="unicode").strip()) - if '@GALAXY_VERSION@' not in [n.attrib.get('name') for n in xml_tree.iter('token')]: + if wrapper_token not in [n.attrib.get('name') for n in xml_tree.iter('token')]: tags_to_update['update_tool'] = True for requirement in xml_tree.iter("requirement"): if requirement.text not in updated_main_req: requirement.set('version', updated_version_dict[requirement.text]) tags_to_update['requirements'].append(ET.tostring(requirement, encoding="unicode").strip()) + print(tags_to_update) write_to_xml(tool_path, xml_tree, tags_to_update) return xml_tree @@ -119,6 +137,7 @@ def write_to_xml(tool_path, xml_tree, tags_to_update): """ Write modified XML to tool_path """ + print(tags_to_update) with open(tool_path, 'r+') as f: xml_text = f.read() for token in tags_to_update['tokens']: @@ -130,7 +149,7 @@ def write_to_xml(tool_path, xml_tree, tags_to_update): # if '@GALAXY_VERSION@' not in tags_to_update['tokens']: if tags_to_update.get('update_tool'): # update the version directly in the tool tag - xml_text = re.sub(r'version="@TOOL_VERSION@\+galaxy.*"', 'version="@TOOL_VERSION@+galaxy0"', xml_text) + xml_text = re.sub(r'version="@TOOL_VERSION@\+galaxy.*?"', 'version="@TOOL_VERSION@+galaxy0"', xml_text) f.seek(0) f.truncate() diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 9bdd43501..c33d38b6f 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -10,6 +10,7 @@ ) from planemo.io import ( coalesce_return_codes, + error, info ) from planemo.tools import ( @@ -38,13 +39,17 @@ def dry_run_option(): @dry_run_option() @command_function def cli(ctx, paths, **kwds): - """Auto-update tool requirements by checking against Conda and updating if newer versions are available""" + """Auto-update tool requirements by checking against Conda and updating if newer versions are available.""" assert_tools = kwds.get("assert_tools", True) recursive = kwds.get("recursive", False) exit_codes = [] + # print([t for t in yield_tool_sources_on_paths(ctx, paths, recursive)]) for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): info("Auto-updating tool %s" % tool_path) - tool_xml = autoupdate.autoupdate(ctx, tool_path, **kwds) + try: + tool_xml = autoupdate.autoupdate(ctx, tool_path, **kwds) + except Exception as e: + error("{} could not be updated - the following error was raised: {}".format(tool_path, e.__str__())) if handle_tool_load_error(tool_path, tool_xml): exit_codes.append(EXIT_CODE_GENERIC_FAILURE) continue From 6536192d71aca0e5cb21abbd45a1af6fff636cd7 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 20 Aug 2020 17:11:23 +0200 Subject: [PATCH 16/39] rewrite code --- planemo/autoupdate.py | 267 +++++++++++++++++++++++------------------- 1 file changed, 145 insertions(+), 122 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 0bf7228da..2879a1a1c 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -1,6 +1,7 @@ """Autoupdate older conda dependencies in the requirements section.""" from __future__ import absolute_import +import collections import re import xml.etree.ElementTree as ET @@ -10,152 +11,174 @@ from planemo.io import error, info -def autoupdate(ctx, tool_path, **kwds): - """ - Autoupdate an XML file - """ - xml_tree = ET.parse(tool_path) - requirements = find_requirements(xml_tree) - for macro_import in requirements['imports']: - # recursively check macros - macro_requirements = autoupdate(ctx, '/'.join(tool_path.split('/')[:-1] + [macro_import]), **kwds) #kwds.get('dry_run')) - for requirement in macro_requirements: - if requirement not in requirements: - requirements[requirement] = macro_requirements[requirement] - if not requirements.get('@TOOL_VERSION@'): - # if tool_version is not specified, finish without changes - error("The @TOOL_VERSION@ token is not specified in {}. This is required for autoupdating.".format(tool_path)) - return requirements - updated_main_req = get_latest_versions(ctx, {requirements.get('main_req'): requirements.get('@TOOL_VERSION@')}, **kwds) - if updated_main_req[requirements.get('main_req')] == requirements.get('@TOOL_VERSION@'): - # check main_req is up-to-date; if so, finish without changes - info("No updates required or made to {}.".format(tool_path)) - return requirements - - if kwds.get('dry_run'): - error("Update required to {}! Tool main requirement has version {}, newest conda version is {}".format( - tool_path, requirements.get('@TOOL_VERSION@'), updated_main_req[requirements.get('main_req')])) - return requirements - - # if main_req is not up-to-date, update everything - updated_version_dict = get_latest_versions(ctx, requirements.get('other_reqs'), **kwds) - update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_req) - info("Tool {} updated.".format(tool_path)) - return requirements - - -def find_requirements(xml_tree): - """ - Get all requirements with versions as a dictionary of the form - {'main_req': main requirement, 'other_reqs': {req1: version, ... }, - 'imports: ['macros.xml'], '*VERSION@': '...'} - """ - requirements = {'other_reqs': {}, 'imports': []} - - # get tokens - for token in xml_tree.iter("token"): - if 'VERSION@' in token.attrib.get('name', ''): - requirements[token.attrib['name']] = token.text +def find_macros(xml_tree): + macros = [] for macro_import in xml_tree.iter("import"): - requirements['imports'].append(macro_import.text) + macros.append(macro_import.text) + if macros: + return macros + else: + return None + - # get requirements +def get_requirements(xml_tree): + requirements = {} + main_req = None for requirement in xml_tree.iter("requirement"): if requirement.attrib.get('version') == '@TOOL_VERSION@': - requirements['main_req'] = requirement.text + main_req = requirement.text else: - requirements['other_reqs'][requirement.text] = requirement.attrib.get('version') - return requirements + requirements[requirement.text] = requirement.attrib.get('version') + return requirements, main_req -def get_latest_versions(ctx, version_dict, **kwds): +def get_tokens(xml_tree): + tokens = {} + for token in xml_tree.iter("token"): + tokens[token.attrib['name']] = token.text + return tokens + + +def check_conda(tool_name, tool_version, ctx, **kwds): """ Update a dict with current conda versions for tool requirements """ - for package in version_dict.keys(): - conda_context = planemo.conda.build_conda_context(ctx, **kwds) - target = planemo.conda.conda_util.CondaTarget(package) - search_results = conda_util.best_search_result(target, conda_context=conda_context) - if search_results[1]: - version_dict[package] = search_results[0]['version'] - else: - error('Requirements could not be found!') - exit() - return version_dict + conda_context = planemo.conda.build_conda_context(ctx, **kwds) + target = planemo.conda.conda_util.CondaTarget(tool_name) + search_results = conda_util.best_search_result(target, conda_context=conda_context) + if search_results[0]['version'] == tool_version: + return None + else: + return search_results[0]['version'] -def update_requirements(tool_path, xml_tree, updated_version_dict, updated_main_req): +def update_xml(tool_path, xml_tree, tags_to_update): """ - Update requirements to latest conda versions - and update version tokens + Write modified XML to tool_path """ - tags_to_update = {'tokens': [], 'requirements': []} - - # get name of token which defines the wrapper version; if just an integer, None - versions = xml_tree.getroot().attrib.get('version')#.split('+galaxy') - if versions: - versions = versions.split('+galaxy') - if len(versions) == 1: - error('Tool version is not specified as two parts separated by +galaxy as required by autoupdate.') - exit() - elif versions[0] != '@TOOL_VERSION@': - error('Tool version does not contain @TOOL_VERSION@ as required by autoupdate.') - exit() - else: - if versions[1][0] == versions[1][-1] == '@': - wrapper_token = versions[1] - else: - wrapper_token = None - else: - wrapper_token = None - print(wrapper_token) + print(tool_path) + print(tags_to_update) - for token in xml_tree.iter("token"): - if token.attrib.get('name') == '@TOOL_VERSION@': - # check this - token.text = list(updated_main_req.values())[0] - elif token.attrib.get('name') == wrapper_token: - token.text = '0' - else: - continue - tags_to_update['tokens'].append(ET.tostring(token, encoding="unicode").strip()) + def update_token(xml_text, token_name, token_value): + return re.sub('{}>.*<{}'.format(*re.split('>.*<', token)), token, xml_text) - if wrapper_token not in [n.attrib.get('name') for n in xml_tree.iter('token')]: - tags_to_update['update_tool'] = True + def update_requirement(xml_text, requirement_name, requirement_value): + return - for requirement in xml_tree.iter("requirement"): - if requirement.text not in updated_main_req: - requirement.set('version', updated_version_dict[requirement.text]) - tags_to_update['requirements'].append(ET.tostring(requirement, encoding="unicode").strip()) - print(tags_to_update) - write_to_xml(tool_path, xml_tree, tags_to_update) - return xml_tree + with open(tool_path, 'r+') as f: + xml_text = f.read() + + for tag_to_update in tags_to_update: + if tag_to_update['type'] = 'token': + xml_text = update_token(xml_text, tag_to_update['key'], tag_to_update['value']) + if tag_to_update['type'] = 'requirement': + update_requirement(xml_text, tag_to_update['key'], tag_to_update['value']) + + + + + # print(tags_to_update) + # with open(tool_path, 'r+') as f: + # xml_text = f.read() + # for token in tags_to_update['tokens']: + # xml_text = re.sub('{}>.*<{}'.format(*re.split('>.*<', token)), token, xml_text) + + # for requirement in tags_to_update['requirements']: + # xml_text = re.sub('{}version=".*"{}'.format(*re.split('version=".*"', requirement)), requirement, xml_text) + # # if '@GALAXY_VERSION@' not in tags_to_update['tokens']: + # if tags_to_update.get('update_tool'): + # # update the version directly in the tool tag + # xml_text = re.sub(r'version="@TOOL_VERSION@\+galaxy.*?"', 'version="@TOOL_VERSION@+galaxy0"', xml_text) -def write_to_xml(tool_path, xml_tree, tags_to_update): + # f.seek(0) + # f.truncate() + # f.write(xml_text) + +def autoupdate(ctx, tool_path, **kwds): """ - Write modified XML to tool_path + Autoupdate an XML file """ - print(tags_to_update) - with open(tool_path, 'r+') as f: - xml_text = f.read() - for token in tags_to_update['tokens']: - xml_text = re.sub('{}>.*<{}'.format(*re.split('>.*<', token)), token, xml_text) + # create a dict of all files that need editing - wrapper plus macros + xml_files = {tool_path: ET.parse(tool_path)} + for macro in find_macros(xml_files[tool_path]): + macro_path = '/'.join(tool_path.split('/')[:-1] + [macro]) + xml_files[macro_path] = ET.parse(macro_path) + + # create a dict of requirements + requirements = {} #collections.defaultdict(dict) + main_req = None + for k, v in xml_files.items(): + r, mr = get_requirements(v) + requirements[k] = r + if mr: + if main_req: + error('Multiple requirements use the token @TOOL_VERSION@!') + main_req = (mr, k) #{mr: k} + if not main_req: + error('No requirement uses the token @TOOL_VERSION@!') + + # create a dict of tokens + tokens = {} + need_to_update = False + xml_to_update = collections.defaultdict(list) + for k, v in xml_files.items(): + tokens[k] = get_tokens(v) + if '@TOOL_VERSION@' in tokens[k]: + check_main_req = check_conda(main_req[0], tokens[k]['@TOOL_VERSION@'], ctx, **kwds) + if check_main_req: + xml_to_update[k].append({'type': 'token', 'key': '@TOOL_VERSION@', 'value': check_main_req}) + need_to_update = True + if not need_to_update: + info("No updates required or made to {}.".format(tool_path)) + return # end here if no update needed + + # check all requirements + for k, v in requirements.items(): + for req in v: + req_check = check_conda(req, v[req], ctx, **kwds) + if req_check: + xml_to_update[k].append({'type': 'requirement', 'tag': req, 'value': req_check}) + + # check all tokens + for k, v in tokens.items(): + if '@GALAXY_VERSION' in v: + xml_to_update[k].append({'type': 'token', 'key': '@GALAXY_VERSION@', 'value': 0}) + + # finally, update each file separately + for k, v in xml_files.items(): + update_xml(k, v, xml_to_update[k]) + + + print(tokens) + print(requirements) + print(xml_to_update) + # requirements[k] = r + # if mr: + # if main_req: + # error('Multiple tools use the token @TOOL_VERSION@!') + # main_req = {mr: k} # (mr, k) + + # tokens = get_tokens(xml_files[tool_path])) + + + # print(requirements) + # print(main_req) + + + + + + - for requirement in tags_to_update['requirements']: - xml_text = re.sub('{}version=".*"{}'.format(*re.split('version=".*"', requirement)), requirement, xml_text) - # if '@GALAXY_VERSION@' not in tags_to_update['tokens']: - if tags_to_update.get('update_tool'): - # update the version directly in the tool tag - xml_text = re.sub(r'version="@TOOL_VERSION@\+galaxy.*?"', 'version="@TOOL_VERSION@+galaxy0"', xml_text) - f.seek(0) - f.truncate() - f.write(xml_text) +# def get_tokens(): +# return ... +# def conda_check(): +# return ... -__all__ = ( - "autoupdate" -) +# def update(): +# return \ No newline at end of file From 6b44dd8025c05df5df8808d8ff923ca846b0f2e1 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Fri, 21 Aug 2020 17:39:19 +0200 Subject: [PATCH 17/39] commit --- docs/autoupdate.rst | 6 +- planemo/autoupdate.py | 167 +++++++++++++---------------- planemo/commands/cmd_autoupdate.py | 58 +++++++++- 3 files changed, 135 insertions(+), 96 deletions(-) diff --git a/docs/autoupdate.rst b/docs/autoupdate.rst index 38e0e6fdc..4f51c15ea 100644 --- a/docs/autoupdate.rst +++ b/docs/autoupdate.rst @@ -12,7 +12,11 @@ Planemo provides a ``autoupdate`` subcommand which can be used to perform this t planemo autoupdate tool_wrapper.xml -There are various flags which can be applied; two of the most important are ``--recursive``, which performs the autoupdate recursively on subdirectories, and ``--dry-run``, which checks if tool requirements are up-to-date without making the necessary change automatically. +There are various flags which can be applied; these are some of the most important: + - ``--recursive``, which performs the autoupdate recursively on subdirectories + - ``--dry-run``, which checks if tool requirements are up-to-date without making the necessary change automatically + - ``--test``, which runs tests on all autoupdated tools + - ``--update_test_data`` (if ``--test`` is also selected) which will update test data for failed tests One of the most efficient ways to use it is by implementing a CI cron job which runs the command on an entire GitHub repository of tools. diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 2879a1a1c..2bb9baa88 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -12,30 +12,36 @@ def find_macros(xml_tree): + """ + Get macros from the XML tree + """ macros = [] for macro_import in xml_tree.iter("import"): macros.append(macro_import.text) - if macros: - return macros - else: - return None + return macros def get_requirements(xml_tree): + """ + Get requirements from the XML tree + """ requirements = {} main_req = None for requirement in xml_tree.iter("requirement"): if requirement.attrib.get('version') == '@TOOL_VERSION@': main_req = requirement.text else: - requirements[requirement.text] = requirement.attrib.get('version') + requirements[requirement.text] = {'tag': ET.tostring(requirement, encoding="unicode").strip(), 'text': requirement.attrib.get('version')} return requirements, main_req def get_tokens(xml_tree): + """ + Get tokens from the XML tree + """ tokens = {} for token in xml_tree.iter("token"): - tokens[token.attrib['name']] = token.text + tokens[token.attrib['name']] = {'tag': ET.tostring(token, encoding="unicode").strip(), 'text': token.text} return tokens @@ -52,70 +58,74 @@ def check_conda(tool_name, tool_version, ctx, **kwds): return search_results[0]['version'] -def update_xml(tool_path, xml_tree, tags_to_update): +def update_xml(tool_path, xml_tree, tags_to_update, wrapper_version_token, is_macro=False): """ Write modified XML to tool_path """ + def update_token(xml_text, tag, token_value): + new_tag = '>{}<'.format(token_value).join(re.split('>.*<', tag)) + return re.sub(tag, new_tag, xml_text) - print(tool_path) - print(tags_to_update) - - def update_token(xml_text, token_name, token_value): - return re.sub('{}>.*<{}'.format(*re.split('>.*<', token)), token, xml_text) - - def update_requirement(xml_text, requirement_name, requirement_value): - return + def update_requirement(xml_text, tag, requirement_value): + new_tag = 'version="{}"'.format(requirement_value).join(re.split('version=".*"', tag)) + return re.sub(tag, new_tag, xml_text) with open(tool_path, 'r+') as f: xml_text = f.read() - for tag_to_update in tags_to_update: - if tag_to_update['type'] = 'token': - xml_text = update_token(xml_text, tag_to_update['key'], tag_to_update['value']) - if tag_to_update['type'] = 'requirement': - update_requirement(xml_text, tag_to_update['key'], tag_to_update['value']) - - - - - # print(tags_to_update) - # with open(tool_path, 'r+') as f: - # xml_text = f.read() - # for token in tags_to_update['tokens']: - # xml_text = re.sub('{}>.*<{}'.format(*re.split('>.*<', token)), token, xml_text) - - # for requirement in tags_to_update['requirements']: - # xml_text = re.sub('{}version=".*"{}'.format(*re.split('version=".*"', requirement)), requirement, xml_text) - - # # if '@GALAXY_VERSION@' not in tags_to_update['tokens']: - # if tags_to_update.get('update_tool'): - # # update the version directly in the tool tag - # xml_text = re.sub(r'version="@TOOL_VERSION@\+galaxy.*?"', 'version="@TOOL_VERSION@+galaxy0"', xml_text) - - # f.seek(0) - # f.truncate() - # f.write(xml_text) - -def autoupdate(ctx, tool_path, **kwds): + if tag_to_update['type'] == 'token': + xml_text = update_token(xml_text, tag_to_update['tag'], tag_to_update['value']) + if tag_to_update['type'] == 'requirement': + xml_text = update_requirement(xml_text, tag_to_update['tag'], tag_to_update['value']) + if not wrapper_version_token and not is_macro: + # i.e. @GALAXY_VERSION@ not specified so update the version directly in the tool tag + tool_tag = re.sub('version="@TOOL_VERSION@.*?"', 'version="@TOOL_VERSION@+galaxy0"', + re.findall('', xml_text)[0]) + xml_text = re.sub('', tool_tag, xml_text) + f.seek(0) + f.truncate() + f.write(xml_text) + + +def autoupdate(ctx, tool_path, force_update=False, **kwds): """ Autoupdate an XML file """ # create a dict of all files that need editing - wrapper plus macros xml_files = {tool_path: ET.parse(tool_path)} + + # get name of token which defines the wrapper version; if just an integer, None + versions = xml_files[tool_path].getroot().attrib.get('version') + if versions: + versions = versions.split('+galaxy') + if versions[0] != '@TOOL_VERSION@': + error('Tool version does not contain @TOOL_VERSION@ as required by autoupdate.') + return + elif len(versions) == 1: + wrapper_version_token = None + else: + if versions[1][0] == versions[1][-1] == '@': + wrapper_version_token = versions[1] + else: + wrapper_version_token = None + else: + wrapper_version_token = None + + # add macros to xml_files for macro in find_macros(xml_files[tool_path]): macro_path = '/'.join(tool_path.split('/')[:-1] + [macro]) xml_files[macro_path] = ET.parse(macro_path) # create a dict of requirements - requirements = {} #collections.defaultdict(dict) + requirements = {} main_req = None for k, v in xml_files.items(): - r, mr = get_requirements(v) - requirements[k] = r - if mr: + file_reqs, file_main_req = get_requirements(v) + requirements[k] = file_reqs + if file_main_req: if main_req: error('Multiple requirements use the token @TOOL_VERSION@!') - main_req = (mr, k) #{mr: k} + main_req = (file_main_req, k) if not main_req: error('No requirement uses the token @TOOL_VERSION@!') @@ -125,60 +135,37 @@ def autoupdate(ctx, tool_path, **kwds): xml_to_update = collections.defaultdict(list) for k, v in xml_files.items(): tokens[k] = get_tokens(v) + # check if it is @TOOL_VERSION@ and if so do check_conda if '@TOOL_VERSION@' in tokens[k]: - check_main_req = check_conda(main_req[0], tokens[k]['@TOOL_VERSION@'], ctx, **kwds) + check_main_req = check_conda(main_req[0], tokens[k]['@TOOL_VERSION@']['text'], ctx, **kwds) if check_main_req: - xml_to_update[k].append({'type': 'token', 'key': '@TOOL_VERSION@', 'value': check_main_req}) + xml_to_update[k].append({'type': 'token', 'tag': tokens[k]['@TOOL_VERSION@']['tag'], 'value': check_main_req}) need_to_update = True - if not need_to_update: + if not need_to_update and not force_update: info("No updates required or made to {}.".format(tool_path)) return # end here if no update needed + if kwds.get('dry_run'): + error("Update required to {}! Tool main requirement has version {}, newest conda version is {}".format( + tool_path, tokens[k]['@TOOL_VERSION@']['text'], check_main_req)) + return + # check all requirements for k, v in requirements.items(): for req in v: - req_check = check_conda(req, v[req], ctx, **kwds) + req_check = check_conda(req, v[req]['text'], ctx, **kwds) if req_check: - xml_to_update[k].append({'type': 'requirement', 'tag': req, 'value': req_check}) + xml_to_update[k].append({'type': 'requirement', 'tag': v[req]['tag'], 'value': req_check}) - # check all tokens - for k, v in tokens.items(): - if '@GALAXY_VERSION' in v: - xml_to_update[k].append({'type': 'token', 'key': '@GALAXY_VERSION@', 'value': 0}) + # check all tokens, if wrapper_version_token exists + if wrapper_version_token: + for k, v in tokens.items(): + if wrapper_version_token in v: + xml_to_update[k].append({'type': 'token', 'tag': v[wrapper_version_token]['tag'], 'value': 0}) # finally, update each file separately for k, v in xml_files.items(): - update_xml(k, v, xml_to_update[k]) - - - print(tokens) - print(requirements) - print(xml_to_update) - # requirements[k] = r - # if mr: - # if main_req: - # error('Multiple tools use the token @TOOL_VERSION@!') - # main_req = {mr: k} # (mr, k) - - # tokens = get_tokens(xml_files[tool_path])) - - - # print(requirements) - # print(main_req) - - - - - - - - - -# def get_tokens(): -# return ... - -# def conda_check(): -# return ... + update_xml(k, v, xml_to_update[k], wrapper_version_token, is_macro=(k != tool_path)) -# def update(): -# return \ No newline at end of file + info("Tool {} updated.".format(tool_path)) + return 0 diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index c33d38b6f..82f402002 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -1,9 +1,14 @@ """Module describing the planemo ``autoupdate`` command.""" +import os.path + import click from planemo import autoupdate, options from planemo.cli import command_function from planemo.config import planemo_option +from planemo.engine.test import ( + test_runnables, +) from planemo.exit_codes import ( EXIT_CODE_GENERIC_FAILURE, EXIT_CODE_OK @@ -11,7 +16,11 @@ from planemo.io import ( coalesce_return_codes, error, - info + info, + temp_directory +) +from planemo.runnable import ( + for_paths, ) from planemo.tools import ( is_tool_load_error, @@ -28,15 +37,37 @@ def dry_run_option(): ) +def force_update_option(): + """Update all requirements, even if main requirement has not changed""" + return planemo_option( + "--force-update", + is_flag=True, + help="Update all requirements, even if main requirement has not changed." + ) + + +def test_option(): + """Test updated XML files""" + return planemo_option( + "--test", + is_flag=True, + help="Test updated XML files." + ) + + @click.command('autoupdate') @options.optional_tools_arg(multiple=True) -@options.conda_target_options() +@dry_run_option() +@options.recursive_option() +@force_update_option() +@test_option() +@options.test_options() +@options.galaxy_target_options() +@options.galaxy_config_options() @options.report_level_option() @options.report_xunit() @options.fail_level_option() @options.skip_option() -@options.recursive_option() -@dry_run_option() @command_function def cli(ctx, paths, **kwds): """Auto-update tool requirements by checking against Conda and updating if newer versions are available.""" @@ -44,10 +75,14 @@ def cli(ctx, paths, **kwds): recursive = kwds.get("recursive", False) exit_codes = [] # print([t for t in yield_tool_sources_on_paths(ctx, paths, recursive)]) + dirs_updated = set() for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): info("Auto-updating tool %s" % tool_path) try: + kwds['force_update'] = (os.path.split(tool_path)[0] in dirs_updated) or kwds.get('force_update') tool_xml = autoupdate.autoupdate(ctx, tool_path, **kwds) + if tool_xml == 0: + dirs_updated.add(os.path.split(tool_path)[0]) except Exception as e: error("{} could not be updated - the following error was raised: {}".format(tool_path, e.__str__())) if handle_tool_load_error(tool_path, tool_xml): @@ -55,8 +90,21 @@ def cli(ctx, paths, **kwds): continue else: exit_codes.append(EXIT_CODE_OK) + + if kwds['test']: + info("Running tests for auto-updated tools...") + if not dirs_updated: + info("No tools were updated, so no tests were run.") + else: + with temp_directory(dir=ctx.planemo_directory) as temp_path: + # only test tools in updated directories + paths = [path for path in paths if os.path.split(tool_path)[0] in dirs_updated] + runnables = for_paths(paths, temp_path=temp_path) + kwds["engine"] = "galaxy" + return_value = test_runnables(ctx, runnables, original_paths=paths, **kwds) + return coalesce_return_codes(exit_codes, assert_at_least_one=assert_tools) - ctx.exit() + ctx.exit(return_value) def handle_tool_load_error(tool_path, tool_xml): From c5e6d5ba2aee9a5bc58e30ed057430b03c1d5a14 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 24 Aug 2020 10:37:25 +0200 Subject: [PATCH 18/39] autoupdate --- planemo/autoupdate.py | 6 +++--- planemo/commands/cmd_autoupdate.py | 20 ++++---------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 2bb9baa88..c76f19a3a 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -87,7 +87,7 @@ def update_requirement(xml_text, tag, requirement_value): f.write(xml_text) -def autoupdate(ctx, tool_path, force_update=False, **kwds): +def autoupdate(ctx, tool_path, modified_files=set(), **kwds): """ Autoupdate an XML file """ @@ -141,7 +141,7 @@ def autoupdate(ctx, tool_path, force_update=False, **kwds): if check_main_req: xml_to_update[k].append({'type': 'token', 'tag': tokens[k]['@TOOL_VERSION@']['tag'], 'value': check_main_req}) need_to_update = True - if not need_to_update and not force_update: + if not need_to_update and not (modified_files & set(xml_files)): info("No updates required or made to {}.".format(tool_path)) return # end here if no update needed @@ -168,4 +168,4 @@ def autoupdate(ctx, tool_path, force_update=False, **kwds): update_xml(k, v, xml_to_update[k], wrapper_version_token, is_macro=(k != tool_path)) info("Tool {} updated.".format(tool_path)) - return 0 + return list(xml_files) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 82f402002..11744d9fa 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -37,15 +37,6 @@ def dry_run_option(): ) -def force_update_option(): - """Update all requirements, even if main requirement has not changed""" - return planemo_option( - "--force-update", - is_flag=True, - help="Update all requirements, even if main requirement has not changed." - ) - - def test_option(): """Test updated XML files""" return planemo_option( @@ -59,7 +50,6 @@ def test_option(): @options.optional_tools_arg(multiple=True) @dry_run_option() @options.recursive_option() -@force_update_option() @test_option() @options.test_options() @options.galaxy_target_options() @@ -74,15 +64,13 @@ def cli(ctx, paths, **kwds): assert_tools = kwds.get("assert_tools", True) recursive = kwds.get("recursive", False) exit_codes = [] - # print([t for t in yield_tool_sources_on_paths(ctx, paths, recursive)]) - dirs_updated = set() + modified_files = set() for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): info("Auto-updating tool %s" % tool_path) try: - kwds['force_update'] = (os.path.split(tool_path)[0] in dirs_updated) or kwds.get('force_update') - tool_xml = autoupdate.autoupdate(ctx, tool_path, **kwds) - if tool_xml == 0: - dirs_updated.add(os.path.split(tool_path)[0]) + updated = autoupdate.autoupdate(ctx, tool_path, modified_files=modified_files, **kwds) + if updated: + modified_files += updated except Exception as e: error("{} could not be updated - the following error was raised: {}".format(tool_path, e.__str__())) if handle_tool_load_error(tool_path, tool_xml): From a1a1e3e9f2cc371a3c2b62819d4d3262fc5c9335 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 24 Aug 2020 11:45:12 +0200 Subject: [PATCH 19/39] autoupdate --- planemo/autoupdate.py | 2 +- planemo/commands/cmd_autoupdate.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index c76f19a3a..5c8cfbf40 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -168,4 +168,4 @@ def autoupdate(ctx, tool_path, modified_files=set(), **kwds): update_xml(k, v, xml_to_update[k], wrapper_version_token, is_macro=(k != tool_path)) info("Tool {} updated.".format(tool_path)) - return list(xml_files) + return set(xml_files) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 11744d9fa..2921ca1bd 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -70,7 +70,8 @@ def cli(ctx, paths, **kwds): try: updated = autoupdate.autoupdate(ctx, tool_path, modified_files=modified_files, **kwds) if updated: - modified_files += updated + # modified_files += updated + modified_files.update(updated) except Exception as e: error("{} could not be updated - the following error was raised: {}".format(tool_path, e.__str__())) if handle_tool_load_error(tool_path, tool_xml): From 2cb826297cfb8f5d3362d009110de869e547e216 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 24 Aug 2020 15:20:23 +0200 Subject: [PATCH 20/39] do not add +galaxy0 where not already used, requested by @wm75 --- planemo/autoupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 5c8cfbf40..b61b70cc1 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -77,7 +77,7 @@ def update_requirement(xml_text, tag, requirement_value): xml_text = update_token(xml_text, tag_to_update['tag'], tag_to_update['value']) if tag_to_update['type'] == 'requirement': xml_text = update_requirement(xml_text, tag_to_update['tag'], tag_to_update['value']) - if not wrapper_version_token and not is_macro: + if wrapper_version_token is not None and not is_macro: # i.e. @GALAXY_VERSION@ not specified so update the version directly in the tool tag tool_tag = re.sub('version="@TOOL_VERSION@.*?"', 'version="@TOOL_VERSION@+galaxy0"', re.findall('', xml_text)[0]) @@ -107,7 +107,7 @@ def autoupdate(ctx, tool_path, modified_files=set(), **kwds): if versions[1][0] == versions[1][-1] == '@': wrapper_version_token = versions[1] else: - wrapper_version_token = None + wrapper_version_token = 0 # assume an int else: wrapper_version_token = None From 6bd3a8ad82c3c2849aba818ad38891792a965467 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 7 Sep 2020 13:19:21 +0200 Subject: [PATCH 21/39] restructure autoupdate code, small fixes --- planemo/autoupdate.py | 129 ++++++++++++++++++----------- planemo/commands/cmd_autoupdate.py | 4 +- 2 files changed, 81 insertions(+), 52 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index b61b70cc1..24971328b 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -45,17 +45,17 @@ def get_tokens(xml_tree): return tokens -def check_conda(tool_name, tool_version, ctx, **kwds): +def check_conda(tool_name, ctx, **kwds): """ - Update a dict with current conda versions for tool requirements + Get the most up-to-date conda version for a tool requirement """ conda_context = planemo.conda.build_conda_context(ctx, **kwds) target = planemo.conda.conda_util.CondaTarget(tool_name) search_results = conda_util.best_search_result(target, conda_context=conda_context) - if search_results[0]['version'] == tool_version: - return None - else: - return search_results[0]['version'] + # if search_results[0]['version'] == tool_version: + # return None + # else: + return search_results[0]['version'] def update_xml(tool_path, xml_tree, tags_to_update, wrapper_version_token, is_macro=False): @@ -77,7 +77,7 @@ def update_requirement(xml_text, tag, requirement_value): xml_text = update_token(xml_text, tag_to_update['tag'], tag_to_update['value']) if tag_to_update['type'] == 'requirement': xml_text = update_requirement(xml_text, tag_to_update['tag'], tag_to_update['value']) - if wrapper_version_token is not None and not is_macro: + if wrapper_version_token == 0 and not is_macro: # i.e. @GALAXY_VERSION@ not specified so update the version directly in the tool tag tool_tag = re.sub('version="@TOOL_VERSION@.*?"', 'version="@TOOL_VERSION@+galaxy0"', re.findall('', xml_text)[0]) @@ -87,40 +87,16 @@ def update_requirement(xml_text, tag, requirement_value): f.write(xml_text) -def autoupdate(ctx, tool_path, modified_files=set(), **kwds): +def create_requirement_dict(xml_files): """ - Autoupdate an XML file + Create dict with requirements and find main requirement """ - # create a dict of all files that need editing - wrapper plus macros - xml_files = {tool_path: ET.parse(tool_path)} - - # get name of token which defines the wrapper version; if just an integer, None - versions = xml_files[tool_path].getroot().attrib.get('version') - if versions: - versions = versions.split('+galaxy') - if versions[0] != '@TOOL_VERSION@': - error('Tool version does not contain @TOOL_VERSION@ as required by autoupdate.') - return - elif len(versions) == 1: - wrapper_version_token = None - else: - if versions[1][0] == versions[1][-1] == '@': - wrapper_version_token = versions[1] - else: - wrapper_version_token = 0 # assume an int - else: - wrapper_version_token = None - - # add macros to xml_files - for macro in find_macros(xml_files[tool_path]): - macro_path = '/'.join(tool_path.split('/')[:-1] + [macro]) - xml_files[macro_path] = ET.parse(macro_path) - - # create a dict of requirements requirements = {} main_req = None for k, v in xml_files.items(): + # print(k, v) file_reqs, file_main_req = get_requirements(v) + # print(file_reqs, file_main_req) requirements[k] = file_reqs if file_main_req: if main_req: @@ -128,33 +104,38 @@ def autoupdate(ctx, tool_path, modified_files=set(), **kwds): main_req = (file_main_req, k) if not main_req: error('No requirement uses the token @TOOL_VERSION@!') + return requirements, main_req + - # create a dict of tokens +def create_token_dict(ctx, xml_files, main_req, **kwds): + """ + Create dict with relevant tokens and check conda requirements for main + """ tokens = {} - need_to_update = False + current_main_req, updated_main_req = None, None xml_to_update = collections.defaultdict(list) for k, v in xml_files.items(): tokens[k] = get_tokens(v) # check if it is @TOOL_VERSION@ and if so do check_conda if '@TOOL_VERSION@' in tokens[k]: - check_main_req = check_conda(main_req[0], tokens[k]['@TOOL_VERSION@']['text'], ctx, **kwds) - if check_main_req: - xml_to_update[k].append({'type': 'token', 'tag': tokens[k]['@TOOL_VERSION@']['tag'], 'value': check_main_req}) - need_to_update = True - if not need_to_update and not (modified_files & set(xml_files)): - info("No updates required or made to {}.".format(tool_path)) - return # end here if no update needed + current_main_req = tokens[k]['@TOOL_VERSION@']['text'] + updated_main_req = check_conda(main_req[0], ctx, **kwds) + if current_main_req: + xml_to_update[k].append({'type': 'token', 'tag': tokens[k]['@TOOL_VERSION@']['tag'], 'value': updated_main_req}) + + return tokens, xml_to_update, current_main_req, updated_main_req - if kwds.get('dry_run'): - error("Update required to {}! Tool main requirement has version {}, newest conda version is {}".format( - tool_path, tokens[k]['@TOOL_VERSION@']['text'], check_main_req)) - return +def perform_required_update(ctx, xml_files, tool_path, requirements, tokens, xml_to_update, wrapper_version_token, **kwds): + """ + Carry out the update, if requirements are out-of-date + """ # check all requirements for k, v in requirements.items(): for req in v: - req_check = check_conda(req, v[req]['text'], ctx, **kwds) - if req_check: + req_check = check_conda(req, ctx, **kwds) + # print(req_check, v[req]['text']) + if req_check != v[req]['text']: xml_to_update[k].append({'type': 'requirement', 'tag': v[req]['tag'], 'value': req_check}) # check all tokens, if wrapper_version_token exists @@ -163,9 +144,57 @@ def autoupdate(ctx, tool_path, modified_files=set(), **kwds): if wrapper_version_token in v: xml_to_update[k].append({'type': 'token', 'tag': v[wrapper_version_token]['tag'], 'value': 0}) + # print(xml_to_update) # finally, update each file separately for k, v in xml_files.items(): update_xml(k, v, xml_to_update[k], wrapper_version_token, is_macro=(k != tool_path)) info("Tool {} updated.".format(tool_path)) return set(xml_files) + + +def autoupdate(ctx, tool_path, modified_files=set(), **kwds): + """ + Autoupdate an XML file + """ + # create a dict of all files that need editing - wrapper plus macros + xml_files = {tool_path: ET.parse(tool_path)} + + # get name of token which defines the wrapper version; if just an integer, None + versions = xml_files[tool_path].getroot().attrib.get('version') + if versions: + versions = versions.split('+galaxy') + if versions[0] != '@TOOL_VERSION@': + error('Tool version does not contain @TOOL_VERSION@ as required by autoupdate.') + return + elif len(versions) == 1: + wrapper_version_token = None + else: + if versions[1][0] == versions[1][-1] == '@': + wrapper_version_token = versions[1] + else: + wrapper_version_token = 0 # assume an int + else: + wrapper_version_token = None + + # add macros to xml_files + for macro in find_macros(xml_files[tool_path]): + macro_path = '/'.join(tool_path.split('/')[:-1] + [macro]) + xml_files[macro_path] = ET.parse(macro_path) + + # print(xml_files) + requirements, main_req = create_requirement_dict(xml_files) + # print(requirements) + tokens, xml_to_update, current_main_req, updated_main_req = create_token_dict(ctx, xml_files, main_req, **kwds) + + if current_main_req == updated_main_req and not (modified_files & set(xml_files)): + info("No updates required or made to {}.".format(tool_path)) + return # end here if no update needed + + if kwds.get('dry_run'): + error("Update required to {}! Tool main requirement has version {}, newest conda version is {}".format( + tool_path, current_main_req, updated_main_req)) + return + + else: + return perform_required_update(ctx, xml_files, tool_path, requirements, tokens, xml_to_update, wrapper_version_token, **kwds) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 2921ca1bd..712ffe6b0 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -82,12 +82,12 @@ def cli(ctx, paths, **kwds): if kwds['test']: info("Running tests for auto-updated tools...") - if not dirs_updated: + if not modified_files: info("No tools were updated, so no tests were run.") else: with temp_directory(dir=ctx.planemo_directory) as temp_path: # only test tools in updated directories - paths = [path for path in paths if os.path.split(tool_path)[0] in dirs_updated] + paths = [path for path in paths if os.path.split(tool_path)[0] in modified_files] runnables = for_paths(paths, temp_path=temp_path) kwds["engine"] = "galaxy" return_value = test_runnables(ctx, runnables, original_paths=paths, **kwds) From 6185f449aa66abef3d56d611d438f73dbdfdcede Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 7 Sep 2020 14:49:47 +0200 Subject: [PATCH 22/39] add 2 test cases (w and w/o --dry-run) --- tests/data/autoupdate_test.xml | 10 ++++++++ tests/test_cmd_autoupdate.py | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 tests/data/autoupdate_test.xml create mode 100644 tests/test_cmd_autoupdate.py diff --git a/tests/data/autoupdate_test.xml b/tests/data/autoupdate_test.xml new file mode 100644 index 000000000..27fb18c72 --- /dev/null +++ b/tests/data/autoupdate_test.xml @@ -0,0 +1,10 @@ + + + 0.6.0 + 1 + + + xopen + zeroc-ice + + diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py new file mode 100644 index 000000000..3b87314be --- /dev/null +++ b/tests/test_cmd_autoupdate.py @@ -0,0 +1,43 @@ +"""Tests for the ``autoupdate`` command.""" +import os +from shutil import copyfile + +from .test_utils import ( + CliTestCase, + TEST_DATA_DIR +) + + +class CmdAutoupdateTestCase(CliTestCase): + """Container class defining test cases for the ``autoupdate`` command.""" + + # @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + def test_autoupdate_dry_run(self): + """Test autoupdate command with dry run flag.""" + with self._isolate(): + training_init_command = [ + "autoupdate", + "{}/autoupdate_test.xml".format(TEST_DATA_DIR), + "--conda_channels", "bioconda", + "--dry-run" + ] + self._check_exit_code(training_init_command, exit_code=0) + + def test_autoupdate(self): + """Test autoupdate command.""" + copyfile("{}/autoupdate_test.xml".format(TEST_DATA_DIR), "{}/autoupdate_test_tmp.xml".format(TEST_DATA_DIR)) + with self._isolate(): + training_init_command = [ + "autoupdate", + "{}/autoupdate_test_tmp.xml".format(TEST_DATA_DIR), + "--conda_channels", "bioconda" + ] + + self._check_exit_code(training_init_command, exit_code=0) + + with open("{}/autoupdate_test_tmp.xml".format(TEST_DATA_DIR)) as f: + updated_tool = f.readlines() + assert updated_tool[2].strip() == '0.7.3' + assert updated_tool[3].strip() == '0' + assert updated_tool[7].strip() == 'zeroc-ice' + os.remove("{}/autoupdate_test_tmp.xml".format(TEST_DATA_DIR)) From b4795702b57e87cffe31d481ea6de85d100481f6 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 7 Sep 2020 15:01:15 +0200 Subject: [PATCH 23/39] docs linting --- docs/autoupdate.rst | 4 +- docs/modules.rst | 7 -- docs/planemo.rst | 242 -------------------------------------------- 3 files changed, 1 insertion(+), 252 deletions(-) delete mode 100644 docs/modules.rst delete mode 100644 docs/planemo.rst diff --git a/docs/autoupdate.rst b/docs/autoupdate.rst index 4f51c15ea..a40e0c78b 100644 --- a/docs/autoupdate.rst +++ b/docs/autoupdate.rst @@ -1,5 +1,3 @@ -.. _shed: - ============================= Autoupdating tools ============================= @@ -33,7 +31,7 @@ Formatting tools Implementing an autoupdate CI job -============================= +================================= WIP diff --git a/docs/modules.rst b/docs/modules.rst deleted file mode 100644 index 08b78ddd7..000000000 --- a/docs/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -planemo -======= - -.. toctree:: - :maxdepth: 4 - - planemo diff --git a/docs/planemo.rst b/docs/planemo.rst deleted file mode 100644 index 2311a2fc0..000000000 --- a/docs/planemo.rst +++ /dev/null @@ -1,242 +0,0 @@ -planemo package -=============== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - planemo.commands - planemo.conda_verify - planemo.cwl - planemo.database - planemo.engine - planemo.galaxy - planemo.linters - planemo.reports - planemo.shed - planemo.shed2tap - planemo.test - planemo.training - planemo.xml - -Submodules ----------- - -planemo.bioblend module ------------------------ - -.. automodule:: planemo.bioblend - :members: - :undoc-members: - :show-inheritance: - -planemo.ci module ------------------ - -.. automodule:: planemo.ci - :members: - :undoc-members: - :show-inheritance: - -planemo.cli module ------------------- - -.. automodule:: planemo.cli - :members: - :undoc-members: - :show-inheritance: - -planemo.conda module --------------------- - -.. automodule:: planemo.conda - :members: - :undoc-members: - :show-inheritance: - -planemo.conda\_lint module --------------------------- - -.. automodule:: planemo.conda_lint - :members: - :undoc-members: - :show-inheritance: - -planemo.config module ---------------------- - -.. automodule:: planemo.config - :members: - :undoc-members: - :show-inheritance: - -planemo.context module ----------------------- - -.. automodule:: planemo.context - :members: - :undoc-members: - :show-inheritance: - -planemo.deps module -------------------- - -.. automodule:: planemo.deps - :members: - :undoc-members: - :show-inheritance: - -planemo.docker module ---------------------- - -.. automodule:: planemo.docker - :members: - :undoc-members: - :show-inheritance: - -planemo.exit\_codes module --------------------------- - -.. automodule:: planemo.exit_codes - :members: - :undoc-members: - :show-inheritance: - -planemo.git module ------------------- - -.. automodule:: planemo.git - :members: - :undoc-members: - :show-inheritance: - -planemo.github\_util module ---------------------------- - -.. automodule:: planemo.github_util - :members: - :undoc-members: - :show-inheritance: - -planemo.glob module -------------------- - -.. automodule:: planemo.glob - :members: - :undoc-members: - :show-inheritance: - -planemo.io module ------------------ - -.. automodule:: planemo.io - :members: - :undoc-members: - :show-inheritance: - -planemo.lint module -------------------- - -.. automodule:: planemo.lint - :members: - :undoc-members: - :show-inheritance: - -planemo.mulled module ---------------------- - -.. automodule:: planemo.mulled - :members: - :undoc-members: - :show-inheritance: - -planemo.network\_util module ----------------------------- - -.. automodule:: planemo.network_util - :members: - :undoc-members: - :show-inheritance: - -planemo.options module ----------------------- - -.. automodule:: planemo.options - :members: - :undoc-members: - :show-inheritance: - -planemo.runnable module ------------------------ - -.. automodule:: planemo.runnable - :members: - :undoc-members: - :show-inheritance: - -planemo.shed\_lint module -------------------------- - -.. automodule:: planemo.shed_lint - :members: - :undoc-members: - :show-inheritance: - -planemo.templates module ------------------------- - -.. automodule:: planemo.templates - :members: - :undoc-members: - :show-inheritance: - -planemo.tool\_builder module ----------------------------- - -.. automodule:: planemo.tool_builder - :members: - :undoc-members: - :show-inheritance: - -planemo.tool\_lint module -------------------------- - -.. automodule:: planemo.tool_lint - :members: - :undoc-members: - :show-inheritance: - -planemo.tools module --------------------- - -.. automodule:: planemo.tools - :members: - :undoc-members: - :show-inheritance: - -planemo.virtualenv module -------------------------- - -.. automodule:: planemo.virtualenv - :members: - :undoc-members: - :show-inheritance: - -planemo.workflow\_lint module ------------------------------ - -.. automodule:: planemo.workflow_lint - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: planemo - :members: - :undoc-members: - :show-inheritance: From c3b4c0eaf7781f3f89786577e6f110545062538e Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 7 Sep 2020 15:51:00 +0200 Subject: [PATCH 24/39] autoupdate test passing locally, try and get it to run on the CI as well --- tests/test_cmd_autoupdate.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py index 3b87314be..a787bbb43 100644 --- a/tests/test_cmd_autoupdate.py +++ b/tests/test_cmd_autoupdate.py @@ -11,7 +11,6 @@ class CmdAutoupdateTestCase(CliTestCase): """Container class defining test cases for the ``autoupdate`` command.""" - # @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") def test_autoupdate_dry_run(self): """Test autoupdate command with dry run flag.""" with self._isolate(): @@ -25,19 +24,16 @@ def test_autoupdate_dry_run(self): def test_autoupdate(self): """Test autoupdate command.""" - copyfile("{}/autoupdate_test.xml".format(TEST_DATA_DIR), "{}/autoupdate_test_tmp.xml".format(TEST_DATA_DIR)) - with self._isolate(): + with self._isolate() as f: + copyfile("{}/autoupdate_test.xml".format(TEST_DATA_DIR), os.path.join(f, "autoupdate_test.xml")) training_init_command = [ "autoupdate", - "{}/autoupdate_test_tmp.xml".format(TEST_DATA_DIR), + os.path.join(f, "autoupdate_test.xml"), "--conda_channels", "bioconda" ] - self._check_exit_code(training_init_command, exit_code=0) - - with open("{}/autoupdate_test_tmp.xml".format(TEST_DATA_DIR)) as f: - updated_tool = f.readlines() - assert updated_tool[2].strip() == '0.7.3' - assert updated_tool[3].strip() == '0' - assert updated_tool[7].strip() == 'zeroc-ice' - os.remove("{}/autoupdate_test_tmp.xml".format(TEST_DATA_DIR)) + with open(os.path.join(f, "autoupdate_test.xml")) as f: + updated_tool = f.readlines() + assert updated_tool[2].strip() == '0.7.3' + assert updated_tool[3].strip() == '0' + assert updated_tool[7].strip() == 'zeroc-ice' From 739f44cc16e2d5036552fbe039c0ad977e903da6 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 8 Sep 2020 13:29:04 +0200 Subject: [PATCH 25/39] [ci skip] add skiplist option as suggested by @bgruening --- planemo/commands/cmd_autoupdate.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 712ffe6b0..5767c3b9d 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -46,11 +46,21 @@ def test_option(): ) +def skiplist_option(): + """List of XML files to skip""" + return planemo_option( + "--skiplist", + default=None, + help="Skiplist file, containing a list of tools for which autoupdate should be skipped." + ) + + @click.command('autoupdate') @options.optional_tools_arg(multiple=True) @dry_run_option() @options.recursive_option() @test_option() +@skiplist_option() @options.test_options() @options.galaxy_target_options() @options.galaxy_config_options() @@ -65,7 +75,11 @@ def cli(ctx, paths, **kwds): recursive = kwds.get("recursive", False) exit_codes = [] modified_files = set() + tools_to_skip = [line.rstrip() for line in open(kwds['skiplist'])] if kwds['skiplist'] else [] for (tool_path, tool_xml) in yield_tool_sources_on_paths(ctx, paths, recursive): + if tool_path.split('/')[-1] in tools_to_skip: + info("Skipping tool %s" % tool_path) + continue info("Auto-updating tool %s" % tool_path) try: updated = autoupdate.autoupdate(ctx, tool_path, modified_files=modified_files, **kwds) From f27e28298c24ba6e0d66d97f2fcd788cf52d6830 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 8 Sep 2020 13:58:32 +0200 Subject: [PATCH 26/39] another attempt at fixing the test --- tests/data/autoupdate_test.xml | 4 ++-- tests/test_cmd_autoupdate.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/data/autoupdate_test.xml b/tests/data/autoupdate_test.xml index 27fb18c72..880047c33 100644 --- a/tests/data/autoupdate_test.xml +++ b/tests/data/autoupdate_test.xml @@ -1,7 +1,7 @@ - + 0.6.0 - 1 + 1 xopen diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py index a787bbb43..7b457b73b 100644 --- a/tests/test_cmd_autoupdate.py +++ b/tests/test_cmd_autoupdate.py @@ -2,6 +2,8 @@ import os from shutil import copyfile +from planemo import cli + from .test_utils import ( CliTestCase, TEST_DATA_DIR @@ -14,26 +16,26 @@ class CmdAutoupdateTestCase(CliTestCase): def test_autoupdate_dry_run(self): """Test autoupdate command with dry run flag.""" with self._isolate(): - training_init_command = [ + autoupdate_command = [ "autoupdate", "{}/autoupdate_test.xml".format(TEST_DATA_DIR), "--conda_channels", "bioconda", "--dry-run" ] - self._check_exit_code(training_init_command, exit_code=0) + self._check_exit_code(autoupdate_command, exit_code=0) def test_autoupdate(self): """Test autoupdate command.""" with self._isolate() as f: copyfile("{}/autoupdate_test.xml".format(TEST_DATA_DIR), os.path.join(f, "autoupdate_test.xml")) - training_init_command = [ + autoupdate_command = [ "autoupdate", os.path.join(f, "autoupdate_test.xml"), "--conda_channels", "bioconda" ] - self._check_exit_code(training_init_command, exit_code=0) + self._runner.invoke(cli.planemo, autoupdate_command) with open(os.path.join(f, "autoupdate_test.xml")) as f: updated_tool = f.readlines() assert updated_tool[2].strip() == '0.7.3' - assert updated_tool[3].strip() == '0' + assert updated_tool[3].strip() == '0' assert updated_tool[7].strip() == 'zeroc-ice' From 940b577c300e7f35cdb2501b601eeb9fe8846953 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 8 Sep 2020 14:36:15 +0200 Subject: [PATCH 27/39] create xml file for autoupdate test during the test --- tests/data/autoupdate_test.xml | 10 --------- tests/test_cmd_autoupdate.py | 40 +++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 23 deletions(-) delete mode 100644 tests/data/autoupdate_test.xml diff --git a/tests/data/autoupdate_test.xml b/tests/data/autoupdate_test.xml deleted file mode 100644 index 880047c33..000000000 --- a/tests/data/autoupdate_test.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - 0.6.0 - 1 - - - xopen - zeroc-ice - - diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py index 7b457b73b..98e5a8a9f 100644 --- a/tests/test_cmd_autoupdate.py +++ b/tests/test_cmd_autoupdate.py @@ -1,24 +1,38 @@ """Tests for the ``autoupdate`` command.""" -import os -from shutil import copyfile - -from planemo import cli +import tempfile from .test_utils import ( - CliTestCase, - TEST_DATA_DIR + CliTestCase ) +def create_tmp_test_tool_file(): + xml_str = b""" + + 0.6.0 + 1 + + + xopen + zeroc-ice + + + """ + t = tempfile.NamedTemporaryFile(suffix='.xml', delete=False) + t.write(xml_str) + return t.name + + class CmdAutoupdateTestCase(CliTestCase): """Container class defining test cases for the ``autoupdate`` command.""" def test_autoupdate_dry_run(self): """Test autoupdate command with dry run flag.""" with self._isolate(): + xmlfile = create_tmp_test_tool_file() autoupdate_command = [ "autoupdate", - "{}/autoupdate_test.xml".format(TEST_DATA_DIR), + xmlfile, "--conda_channels", "bioconda", "--dry-run" ] @@ -26,16 +40,16 @@ def test_autoupdate_dry_run(self): def test_autoupdate(self): """Test autoupdate command.""" - with self._isolate() as f: - copyfile("{}/autoupdate_test.xml".format(TEST_DATA_DIR), os.path.join(f, "autoupdate_test.xml")) + with self._isolate(): + xmlfile = create_tmp_test_tool_file() autoupdate_command = [ "autoupdate", - os.path.join(f, "autoupdate_test.xml"), + xmlfile, "--conda_channels", "bioconda" ] - self._runner.invoke(cli.planemo, autoupdate_command) - with open(os.path.join(f, "autoupdate_test.xml")) as f: - updated_tool = f.readlines() + self._check_exit_code(autoupdate_command, exit_code=0) + with open(xmlfile) as stream: + updated_tool = stream.readlines() assert updated_tool[2].strip() == '0.7.3' assert updated_tool[3].strip() == '0' assert updated_tool[7].strip() == 'zeroc-ice' From 08e59881f3d7628320c51e16d03428d0e3f5a745 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Fri, 25 Sep 2020 18:49:50 +0200 Subject: [PATCH 28/39] change tests to check stdout --- tests/test_cmd_autoupdate.py | 39 ++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py index 98e5a8a9f..b47df3818 100644 --- a/tests/test_cmd_autoupdate.py +++ b/tests/test_cmd_autoupdate.py @@ -6,19 +6,18 @@ ) -def create_tmp_test_tool_file(): - xml_str = b""" +def create_tmp_test_tool_file(tool_version): + xml_str = """ - 0.6.0 + {} 1 xopen - zeroc-ice - """ - t = tempfile.NamedTemporaryFile(suffix='.xml', delete=False) + """.format(tool_version) + t = tempfile.NamedTemporaryFile(suffix='.xml', delete=False, mode='w') t.write(xml_str) return t.name @@ -29,27 +28,37 @@ class CmdAutoupdateTestCase(CliTestCase): def test_autoupdate_dry_run(self): """Test autoupdate command with dry run flag.""" with self._isolate(): - xmlfile = create_tmp_test_tool_file() + xmlfile = create_tmp_test_tool_file('0.6.0') autoupdate_command = [ "autoupdate", xmlfile, "--conda_channels", "bioconda", "--dry-run" ] - self._check_exit_code(autoupdate_command, exit_code=0) + result = self._runner.invoke(self._cli.planemo, autoupdate_command) + assert "Update required to {}!".format(xmlfile) in result.output + assert "Tool main requirement has version 0.6.0, newest conda version is 0.7.3" in result.output def test_autoupdate(self): """Test autoupdate command.""" with self._isolate(): - xmlfile = create_tmp_test_tool_file() + xmlfile = create_tmp_test_tool_file('0.6.0') autoupdate_command = [ "autoupdate", xmlfile, "--conda_channels", "bioconda" ] - self._check_exit_code(autoupdate_command, exit_code=0) - with open(xmlfile) as stream: - updated_tool = stream.readlines() - assert updated_tool[2].strip() == '0.7.3' - assert updated_tool[3].strip() == '0' - assert updated_tool[7].strip() == 'zeroc-ice' + result = self._runner.invoke(self._cli.planemo, autoupdate_command) + assert 'Tool {} updated.'.format(xmlfile) in result.output + + def test_autoupdate_no_update_needed(self): + """Test autoupdate command when no update is needed.""" + with self._isolate(): + xmlfile = create_tmp_test_tool_file('0.7.3') + autoupdate_command = [ + "autoupdate", + xmlfile, + "--conda_channels", "bioconda" + ] + result = self._runner.invoke(self._cli.planemo, autoupdate_command) + assert 'No updates required or made to {}.'.format(xmlfile) in result.output From 36ac70986e375d0576f37c30e3b48e5f9a0b3712 Mon Sep 17 00:00:00 2001 From: simonbray Date: Fri, 25 Sep 2020 21:11:20 +0200 Subject: [PATCH 29/39] ensure conda is installed --- planemo/autoupdate.py | 2 ++ tests/test_cmd_autoupdate.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 24971328b..198cfabb1 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -50,6 +50,8 @@ def check_conda(tool_name, ctx, **kwds): Get the most up-to-date conda version for a tool requirement """ conda_context = planemo.conda.build_conda_context(ctx, **kwds) + if not conda_context.is_conda_installed(): + error("Conda is not installed! Try running planemo conda_init.") target = planemo.conda.conda_util.CondaTarget(tool_name) search_results = conda_util.best_search_result(target, conda_context=conda_context) # if search_results[0]['version'] == tool_version: diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py index b47df3818..38747d148 100644 --- a/tests/test_cmd_autoupdate.py +++ b/tests/test_cmd_autoupdate.py @@ -1,6 +1,8 @@ """Tests for the ``autoupdate`` command.""" import tempfile +from planemo.commands.cmd_conda_init import cli as conda_init + from .test_utils import ( CliTestCase ) @@ -25,6 +27,11 @@ def create_tmp_test_tool_file(tool_version): class CmdAutoupdateTestCase(CliTestCase): """Container class defining test cases for the ``autoupdate`` command.""" + def setUp(self): + super(CmdAutoupdateTestCase, self).setUp() + r = self._runner.invoke(self._cli.planemo, ['conda_init']) + # assert r.output == '' + def test_autoupdate_dry_run(self): """Test autoupdate command with dry run flag.""" with self._isolate(): From f13cce14dff906526eaa4453b63eb209f114eec2 Mon Sep 17 00:00:00 2001 From: simonbray Date: Fri, 25 Sep 2020 21:17:55 +0200 Subject: [PATCH 30/39] lint --- tests/test_cmd_autoupdate.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py index 38747d148..84306ba63 100644 --- a/tests/test_cmd_autoupdate.py +++ b/tests/test_cmd_autoupdate.py @@ -1,8 +1,6 @@ """Tests for the ``autoupdate`` command.""" import tempfile -from planemo.commands.cmd_conda_init import cli as conda_init - from .test_utils import ( CliTestCase ) @@ -29,8 +27,7 @@ class CmdAutoupdateTestCase(CliTestCase): def setUp(self): super(CmdAutoupdateTestCase, self).setUp() - r = self._runner.invoke(self._cli.planemo, ['conda_init']) - # assert r.output == '' + self._runner.invoke(self._cli.planemo, ['conda_init']) def test_autoupdate_dry_run(self): """Test autoupdate command with dry run flag.""" From c5af2aff5748d46ea3d4dce42e383fbf933c4400 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 29 Sep 2020 10:42:21 +0200 Subject: [PATCH 31/39] (re)add docs --- docs/commands/autoupdate.rst | 216 ++++++++++++++++++++++++++++++ docs/modules.rst | 7 + docs/planemo.rst | 249 +++++++++++++++++++++++++++++++++++ 3 files changed, 472 insertions(+) create mode 100644 docs/commands/autoupdate.rst create mode 100644 docs/modules.rst create mode 100644 docs/planemo.rst diff --git a/docs/commands/autoupdate.rst b/docs/commands/autoupdate.rst new file mode 100644 index 000000000..c7160e154 --- /dev/null +++ b/docs/commands/autoupdate.rst @@ -0,0 +1,216 @@ + +``autoupdate`` command +====================================== + +This section is auto-generated from the help text for the planemo command +``autoupdate``. This help message can be generated with ``planemo autoupdate +--help``. + +**Usage**:: + + planemo autoupdate [OPTIONS] TOOL_PATH + +**Help** + +Auto-update tool requirements by checking against Conda and updating if newer versions are available. +**Options**:: + + + --dry-run Perform a dry run autoupdate without modifying + the XML files. + + -r, --recursive Recursively perform command for + subdirectories. + + --test Test updated XML files. + --skiplist TEXT Skiplist file, containing a list of tools for + which autoupdate should be skipped. + + --update_test_data Update test-data directory with job outputs + (normally written to directory + --job_output_files if specified.) + + --paste_test_data_paths / --no_paste_test_data_paths + By default Planemo will use or not use + Galaxy's path paste option to load test data + into a history based on the engine type it is + targeting. This can override the logic to + explicitly enable or disable path pasting. + + --test_output PATH Output test report (HTML - for humans) + defaults to tool_test_output.html. + + --test_output_text PATH Output test report (Basic text - for display + in CI) + + --test_output_markdown PATH Output test report (Markdown style - for + humans & computers) + + --test_output_xunit PATH Output test report (xunit style - for CI + systems + + --test_output_junit PATH Output test report (jUnit style - for CI + systems + + --test_output_json PATH Output test report (planemo json) defaults to + tool_test_output.json. + + --job_output_files DIRECTORY Write job outputs to specified directory. + --summary [none|minimal|compact] + Summary style printed to planemo's standard + output (see output reports for more complete + summary). Set to 'none' to disable completely. + + --galaxy_root DIRECTORY Root of development galaxy directory to + execute command with. + + --galaxy_python_version [3|3.5|3.6|3.7|3.8] + Python version to start Galaxy under + --extra_tools PATH Extra tool sources to include in Galaxy's tool + panel (file or directory). These will not be + linted/tested/etc... but they will be + available to workflows and for interactive + use. + + --install_galaxy Download and configure a disposable copy of + Galaxy from github. + + --galaxy_branch TEXT Branch of Galaxy to target (defaults to + master) if a Galaxy root isn't specified. + + --galaxy_source TEXT Git source of Galaxy to target (defaults to + the official galaxyproject github source if a + Galaxy root isn't specified. + + --skip_venv Do not create or source a virtualenv + environment for Galaxy, this should be used to + preserve an externally configured virtual + environment or conda environment. + + --no_cache_galaxy Skip caching of Galaxy source and dependencies + obtained with --install_galaxy. Not caching + this results in faster downloads (no git) - so + is better on throw away instances such with + TravisCI. + + --no_cleanup Do not cleanup temp files created for and by + Galaxy. + + --galaxy_email TEXT E-mail address to use when launching single- + user Galaxy server. + + --docker / --no_docker Run Galaxy tools in Docker if enabled. + --docker_cmd TEXT Command used to launch docker (defaults to + docker). + + --docker_sudo / --no_docker_sudo + Flag to use sudo when running docker. + --docker_host TEXT Docker host to target when executing docker + commands (defaults to localhost). + + --docker_sudo_cmd TEXT sudo command to use when --docker_sudo is + enabled (defaults to sudo). + + --mulled_containers, --biocontainers + Test tools against mulled containers (forces + --docker). + + --job_config_file FILE Job configuration file for Galaxy to target. + --tool_dependency_dir DIRECTORY + Tool dependency dir for Galaxy to target. + --test_data DIRECTORY test-data directory to for specified tool(s). + --tool_data_table PATH tool_data_table_conf.xml file to for specified + tool(s). + + --dependency_resolvers_config_file FILE + Dependency resolver configuration for Galaxy + to target. + + --brew_dependency_resolution Configure Galaxy to use plain brew dependency + resolution. + + --shed_dependency_resolution Configure Galaxy to use brewed Tool Shed + dependency resolution. + + --no_dependency_resolution Configure Galaxy with no dependency resolvers. + --conda_prefix DIRECTORY Conda prefix to use for conda dependency + commands. + + --conda_exec FILE Location of conda executable. + --conda_debug Enable more verbose conda logging. + --conda_channels, --conda_ensure_channels TEXT + Ensure conda is configured with specified + comma separated list of channels. + + --conda_use_local Use locally built packages while building + Conda environments. + + --conda_dependency_resolution Configure Galaxy to use only conda for + dependency resolution. + + --conda_copy_dependencies Conda dependency resolution for Galaxy will + copy dependencies instead of attempting to + link them. + + --conda_auto_install / --no_conda_auto_install + Conda dependency resolution for Galaxy will + attempt to install requested but missing + packages. + + --conda_auto_init / --no_conda_auto_init + Conda dependency resolution for Galaxy will + auto install conda itself using miniconda if + not availabe on conda_prefix. + + --profile TEXT Name of profile (created with the + profile_create command) to use with this + command. + + --postgres Use postgres database type. + --database_type [postgres|postgres_docker|sqlite|auto] + Type of database to use for profile - 'auto', + 'sqlite', 'postgres', and 'postgres_docker' + are available options. Use postgres to use an + existing postgres server you user can access + without a password via the psql command. Use + postgres_docker to have Planemo manage a + docker container running postgres. Data with + postgres_docker is not yet persisted past when + you restart the docker container launched by + Planemo so be careful with this option. + + --postgres_psql_path TEXT Name or or path to postgres client binary + (psql). + + --postgres_database_user TEXT Postgres username for managed development + databases. + + --postgres_database_host TEXT Postgres host name for managed development + databases. + + --postgres_database_port TEXT Postgres port for managed development + databases. + + --file_path DIRECTORY Location for files created by Galaxy (e.g. + database/files). + + --database_connection TEXT Database connection string to use for Galaxy. + --shed_tool_conf TEXT Location of shed tools conf file for Galaxy. + --shed_tool_path TEXT Location of shed tools directory for Galaxy. + --galaxy_single_user / --no_galaxy_single_user + By default Planemo will configure Galaxy to + run in single-user mode where there is just + one user and this user is automatically logged + it. Use --no_galaxy_single_user to prevent + Galaxy from running this way. + + --report_level [all|warn|error] + --report_xunit PATH Output an XUnit report, useful for CI testing + --fail_level [warn|error] + -s, --skip TEXT Comma-separated list of lint tests to skip + (e.g. passing --skip 'citations,xml_order' + would skip linting of citations and best- + practice XML ordering. + + --help Show this message and exit. + diff --git a/docs/modules.rst b/docs/modules.rst new file mode 100644 index 000000000..08b78ddd7 --- /dev/null +++ b/docs/modules.rst @@ -0,0 +1,7 @@ +planemo +======= + +.. toctree:: + :maxdepth: 4 + + planemo diff --git a/docs/planemo.rst b/docs/planemo.rst new file mode 100644 index 000000000..7c6f0a2f9 --- /dev/null +++ b/docs/planemo.rst @@ -0,0 +1,249 @@ +planemo package +=============== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + planemo.commands + planemo.conda_verify + planemo.cwl + planemo.database + planemo.engine + planemo.galaxy + planemo.linters + planemo.reports + planemo.shed + planemo.shed2tap + planemo.test + planemo.training + planemo.xml + +Submodules +---------- + +planemo.autoupdate module +------------------------- + +.. automodule:: planemo.autoupdate + :members: + :undoc-members: + :show-inheritance: + +planemo.bioblend module +----------------------- + +.. automodule:: planemo.bioblend + :members: + :undoc-members: + :show-inheritance: + +planemo.ci module +----------------- + +.. automodule:: planemo.ci + :members: + :undoc-members: + :show-inheritance: + +planemo.cli module +------------------ + +.. automodule:: planemo.cli + :members: + :undoc-members: + :show-inheritance: + +planemo.conda module +-------------------- + +.. automodule:: planemo.conda + :members: + :undoc-members: + :show-inheritance: + +planemo.conda\_lint module +-------------------------- + +.. automodule:: planemo.conda_lint + :members: + :undoc-members: + :show-inheritance: + +planemo.config module +--------------------- + +.. automodule:: planemo.config + :members: + :undoc-members: + :show-inheritance: + +planemo.context module +---------------------- + +.. automodule:: planemo.context + :members: + :undoc-members: + :show-inheritance: + +planemo.deps module +------------------- + +.. automodule:: planemo.deps + :members: + :undoc-members: + :show-inheritance: + +planemo.docker module +--------------------- + +.. automodule:: planemo.docker + :members: + :undoc-members: + :show-inheritance: + +planemo.exit\_codes module +-------------------------- + +.. automodule:: planemo.exit_codes + :members: + :undoc-members: + :show-inheritance: + +planemo.git module +------------------ + +.. automodule:: planemo.git + :members: + :undoc-members: + :show-inheritance: + +planemo.github\_util module +--------------------------- + +.. automodule:: planemo.github_util + :members: + :undoc-members: + :show-inheritance: + +planemo.glob module +------------------- + +.. automodule:: planemo.glob + :members: + :undoc-members: + :show-inheritance: + +planemo.io module +----------------- + +.. automodule:: planemo.io + :members: + :undoc-members: + :show-inheritance: + +planemo.lint module +------------------- + +.. automodule:: planemo.lint + :members: + :undoc-members: + :show-inheritance: + +planemo.mulled module +--------------------- + +.. automodule:: planemo.mulled + :members: + :undoc-members: + :show-inheritance: + +planemo.network\_util module +---------------------------- + +.. automodule:: planemo.network_util + :members: + :undoc-members: + :show-inheritance: + +planemo.options module +---------------------- + +.. automodule:: planemo.options + :members: + :undoc-members: + :show-inheritance: + +planemo.runnable module +----------------------- + +.. automodule:: planemo.runnable + :members: + :undoc-members: + :show-inheritance: + +planemo.shed\_lint module +------------------------- + +.. automodule:: planemo.shed_lint + :members: + :undoc-members: + :show-inheritance: + +planemo.templates module +------------------------ + +.. automodule:: planemo.templates + :members: + :undoc-members: + :show-inheritance: + +planemo.tool\_builder module +---------------------------- + +.. automodule:: planemo.tool_builder + :members: + :undoc-members: + :show-inheritance: + +planemo.tool\_lint module +------------------------- + +.. automodule:: planemo.tool_lint + :members: + :undoc-members: + :show-inheritance: + +planemo.tools module +-------------------- + +.. automodule:: planemo.tools + :members: + :undoc-members: + :show-inheritance: + +planemo.virtualenv module +------------------------- + +.. automodule:: planemo.virtualenv + :members: + :undoc-members: + :show-inheritance: + +planemo.workflow\_lint module +----------------------------- + +.. automodule:: planemo.workflow_lint + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: planemo + :members: + :undoc-members: + :show-inheritance: From 8c1382f461c77597649023139185aeadc1180bd8 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 14 Jan 2021 15:39:23 +0100 Subject: [PATCH 32/39] do not modify os-dependent newlines when using autoupdate --- planemo/autoupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 198cfabb1..19d572f04 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -72,7 +72,7 @@ def update_requirement(xml_text, tag, requirement_value): new_tag = 'version="{}"'.format(requirement_value).join(re.split('version=".*"', tag)) return re.sub(tag, new_tag, xml_text) - with open(tool_path, 'r+') as f: + with open(tool_path, 'r+', newline='') as f: xml_text = f.read() for tag_to_update in tags_to_update: if tag_to_update['type'] == 'token': From 16e685731aeceda063327d5fd4a19363175600ce Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 14 Jan 2021 18:15:15 +0100 Subject: [PATCH 33/39] add skip_requirements option to autoupdate --- planemo/autoupdate.py | 8 +++----- planemo/commands/cmd_autoupdate.py | 9 +++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 19d572f04..1be7f0907 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -89,7 +89,7 @@ def update_requirement(xml_text, tag, requirement_value): f.write(xml_text) -def create_requirement_dict(xml_files): +def create_requirement_dict(xml_files, skip_reqs): """ Create dict with requirements and find main requirement """ @@ -99,7 +99,7 @@ def create_requirement_dict(xml_files): # print(k, v) file_reqs, file_main_req = get_requirements(v) # print(file_reqs, file_main_req) - requirements[k] = file_reqs + requirements[k] = [file_req for file_req in file_reqs if file_req not in skip_reqs] if file_main_req: if main_req: error('Multiple requirements use the token @TOOL_VERSION@!') @@ -184,9 +184,7 @@ def autoupdate(ctx, tool_path, modified_files=set(), **kwds): macro_path = '/'.join(tool_path.split('/')[:-1] + [macro]) xml_files[macro_path] = ET.parse(macro_path) - # print(xml_files) - requirements, main_req = create_requirement_dict(xml_files) - # print(requirements) + requirements, main_req = create_requirement_dict(xml_files, kwds.get('skip_requirements', '').split(',')) tokens, xml_to_update, current_main_req, updated_main_req = create_token_dict(ctx, xml_files, main_req, **kwds) if current_main_req == updated_main_req and not (modified_files & set(xml_files)): diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 5767c3b9d..a5bae3b4e 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -55,6 +55,15 @@ def skiplist_option(): ) +def skip_requirements_option(): + """List of requirements to skip""" + return planemo_option( + "--skip_requirements", + default=None, + help="Comma-separated list of requirements which should be not be updated. Default is python,r-base,perl." + ) + + @click.command('autoupdate') @options.optional_tools_arg(multiple=True) @dry_run_option() From 5b582b522059595043ea38bdc30a09b42a27aeee Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Fri, 12 Mar 2021 13:57:02 +0100 Subject: [PATCH 34/39] fix bug introduced in previous commit --- planemo/autoupdate.py | 4 +--- tests/test_cmd_autoupdate.py | 9 +++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 1be7f0907..9d6b81a42 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -96,10 +96,8 @@ def create_requirement_dict(xml_files, skip_reqs): requirements = {} main_req = None for k, v in xml_files.items(): - # print(k, v) file_reqs, file_main_req = get_requirements(v) - # print(file_reqs, file_main_req) - requirements[k] = [file_req for file_req in file_reqs if file_req not in skip_reqs] + requirements[k] = {k: v for k, v in file_reqs.items() if k not in skip_reqs} if file_main_req: if main_req: error('Multiple requirements use the token @TOOL_VERSION@!') diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py index 84306ba63..af1b811e4 100644 --- a/tests/test_cmd_autoupdate.py +++ b/tests/test_cmd_autoupdate.py @@ -7,6 +7,11 @@ def create_tmp_test_tool_file(tool_version): + """ + Note that to ensure this test is stable, we use packages that have + been migrated from bioconda to conda-forge and test with --conda_channels bioconda + so that the versions are guaranteed not to increase in future. + """ xml_str = """ {} @@ -14,6 +19,7 @@ def create_tmp_test_tool_file(tool_version): xopen + smina """.format(tool_version) @@ -54,6 +60,9 @@ def test_autoupdate(self): ] result = self._runner.invoke(self._cli.planemo, autoupdate_command) assert 'Tool {} updated.'.format(xmlfile) in result.output + with open(xmlfile) as f: + xmlfile_contents = f.read() + assert "2017.11.9" in xmlfile_contents def test_autoupdate_no_update_needed(self): """Test autoupdate command when no update is needed.""" From a5be98f774a6f9ce3b60a43af539ff7546315f7a Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Fri, 12 Mar 2021 20:43:43 +0100 Subject: [PATCH 35/39] add link to autoupdate ci, fix update_test_data --- docs/autoupdate.rst | 4 +++- planemo/commands/cmd_autoupdate.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/autoupdate.rst b/docs/autoupdate.rst index a40e0c78b..c38ea7320 100644 --- a/docs/autoupdate.rst +++ b/docs/autoupdate.rst @@ -33,6 +33,8 @@ Formatting tools Implementing an autoupdate CI job ================================= -WIP + A useful way to use the autoupdate command is to implement it as a CI job, so that tools in a repo can be updated on a regular basis (e.g. weekly). An example implementation is the `planemo-autoupdate`_ GitHub bot. + +`__ .. _IUC best practices: https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices/tool_xml.html diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index a5bae3b4e..3bed02307 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -110,8 +110,8 @@ def cli(ctx, paths, **kwds): else: with temp_directory(dir=ctx.planemo_directory) as temp_path: # only test tools in updated directories - paths = [path for path in paths if os.path.split(tool_path)[0] in modified_files] - runnables = for_paths(paths, temp_path=temp_path) + modified_paths = [path for path in paths if path in modified_files] + runnables = for_paths(modified_paths, temp_path=temp_path) kwds["engine"] = "galaxy" return_value = test_runnables(ctx, runnables, original_paths=paths, **kwds) From bc3ed0c58f0657be52a2c37eadde30b3c7062212 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Sat, 13 Mar 2021 17:15:19 +0100 Subject: [PATCH 36/39] add default to skip_requirements arg --- planemo/commands/cmd_autoupdate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 3bed02307..49dc0a9ec 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -1,6 +1,4 @@ """Module describing the planemo ``autoupdate`` command.""" -import os.path - import click from planemo import autoupdate, options @@ -59,7 +57,7 @@ def skip_requirements_option(): """List of requirements to skip""" return planemo_option( "--skip_requirements", - default=None, + default='python,r-base,perl', help="Comma-separated list of requirements which should be not be updated. Default is python,r-base,perl." ) From 3420eae7b7548ae4ff51bc1e81fe0f38486f11e6 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 15 Mar 2021 19:25:28 +0100 Subject: [PATCH 37/39] fix for testing --- planemo/commands/cmd_autoupdate.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 49dc0a9ec..9e74f4d5d 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -91,7 +91,6 @@ def cli(ctx, paths, **kwds): try: updated = autoupdate.autoupdate(ctx, tool_path, modified_files=modified_files, **kwds) if updated: - # modified_files += updated modified_files.update(updated) except Exception as e: error("{} could not be updated - the following error was raised: {}".format(tool_path, e.__str__())) @@ -102,13 +101,13 @@ def cli(ctx, paths, **kwds): exit_codes.append(EXIT_CODE_OK) if kwds['test']: - info("Running tests for auto-updated tools...") if not modified_files: info("No tools were updated, so no tests were run.") else: with temp_directory(dir=ctx.planemo_directory) as temp_path: # only test tools in updated directories - modified_paths = [path for path in paths if path in modified_files] + modified_paths = [tool_path for tool_path, tool_xml in yield_tool_sources_on_paths(ctx, paths, recursive) if tool_path in modified_files] + info(f"Running tests for the following auto-updated tools: {', '.join(modified_paths)}") runnables = for_paths(modified_paths, temp_path=temp_path) kwds["engine"] = "galaxy" return_value = test_runnables(ctx, runnables, original_paths=paths, **kwds) From 599cb59542baac88ff9822630f93c40a46ecf5ff Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 16 Mar 2021 12:34:26 +0100 Subject: [PATCH 38/39] fix autoupdate skip requirements, update docs, some general tidying --- docs/autoupdate.rst | 14 +++++++------- docs/commands.rst | 1 + docs/commands/autoupdate.rst | 7 ++++++- docs/planemo.commands.rst | 8 ++++++++ docs/planemo.engine.rst | 8 ++++++++ planemo/autoupdate.py | 6 +----- planemo/commands/cmd_autoupdate.py | 9 ++++----- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/docs/autoupdate.rst b/docs/autoupdate.rst index c38ea7320..09a5420c6 100644 --- a/docs/autoupdate.rst +++ b/docs/autoupdate.rst @@ -13,17 +13,17 @@ Planemo provides a ``autoupdate`` subcommand which can be used to perform this t There are various flags which can be applied; these are some of the most important: - ``--recursive``, which performs the autoupdate recursively on subdirectories - ``--dry-run``, which checks if tool requirements are up-to-date without making the necessary change automatically - - ``--test``, which runs tests on all autoupdated tools + - ``--test``, which runs tests on all autoupdated tools. If this flag is used, all options available for ``planemo test`` are also available. - ``--update_test_data`` (if ``--test`` is also selected) which will update test data for failed tests + - ``--skiplist``, pointing to a list of tool wrappers for which updates should be skipped + - ``--skip_requirements`` with a comma-separated list of packages not to update. ``python``, ``perl``, ``r-base`` are skipped by default. -One of the most efficient ways to use it is by implementing a CI cron job which runs the command on an entire GitHub repository of tools. +One of the most efficient ways to use it is by implementing a CI cron job which runs the command on an entire GitHub repository of tool wrappers. Formatting tools ============================= - ``autoupdate`` assumes that XML tools are formatted in a certain way - in accordance with the `IUC best practices`_ - -`__. In particular: +``autoupdate`` assumes that XML tools are formatted in a certain way - in accordance with the `IUC best practices`_. In particular: - the tool ``version`` attribute must be set to ``@TOOL_VERSION@+galaxy0`` or ``@TOOL_VERSION@+galaxy@GALAXY_VERSION`` - A token ``@TOOL_VERSION@`` should be created which corresponds to the version number of the main requirement. @@ -33,8 +33,8 @@ Formatting tools Implementing an autoupdate CI job ================================= - A useful way to use the autoupdate command is to implement it as a CI job, so that tools in a repo can be updated on a regular basis (e.g. weekly). An example implementation is the `planemo-autoupdate`_ GitHub bot. + A useful way to use the autoupdate command is to implement it as a CI job, so that tools in a repo can be updated on a regular basis (e.g. weekly). An example implementation is the `planemo-autoupdate`_ GitHub bot. -`__ .. _IUC best practices: https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices/tool_xml.html +.. _planemo-autoupdate: https://github.com/planemo-autoupdate/autoupdate \ No newline at end of file diff --git a/docs/commands.rst b/docs/commands.rst index e21daece3..bbf05f05a 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -7,6 +7,7 @@ implemented as a subcommand of the ``planemo`` executable. This section of the documentation describes these commands. +.. include:: commands/autoupdate.rst .. include:: commands/ci_find_repos.rst .. include:: commands/ci_find_tools.rst .. include:: commands/clone.rst diff --git a/docs/commands/autoupdate.rst b/docs/commands/autoupdate.rst index c7160e154..9928d655b 100644 --- a/docs/commands/autoupdate.rst +++ b/docs/commands/autoupdate.rst @@ -26,6 +26,10 @@ Auto-update tool requirements by checking against Conda and updating if newer ve --skiplist TEXT Skiplist file, containing a list of tools for which autoupdate should be skipped. + --skip_requirements TEXT Comma-separated list of requirements which + should be not be updated. Default is + python,r-base,perl. + --update_test_data Update test-data directory with job outputs (normally written to directory --job_output_files if specified.) @@ -52,6 +56,7 @@ Auto-update tool requirements by checking against Conda and updating if newer ve --test_output_junit PATH Output test report (jUnit style - for CI systems + --test_output_allure DIRECTORY Output test allure2 framework resutls --test_output_json PATH Output test report (planemo json) defaults to tool_test_output.json. @@ -64,7 +69,7 @@ Auto-update tool requirements by checking against Conda and updating if newer ve --galaxy_root DIRECTORY Root of development galaxy directory to execute command with. - --galaxy_python_version [3|3.5|3.6|3.7|3.8] + --galaxy_python_version [3|3.6|3.7|3.8|3.9] Python version to start Galaxy under --extra_tools PATH Extra tool sources to include in Galaxy's tool panel (file or directory). These will not be diff --git a/docs/planemo.commands.rst b/docs/planemo.commands.rst index 6239a23d3..ba7c75e04 100644 --- a/docs/planemo.commands.rst +++ b/docs/planemo.commands.rst @@ -4,6 +4,14 @@ planemo.commands package Submodules ---------- +planemo.commands.cmd\_autoupdate module +--------------------------------------- + +.. automodule:: planemo.commands.cmd_autoupdate + :members: + :undoc-members: + :show-inheritance: + planemo.commands.cmd\_ci\_find\_repos module -------------------------------------------- diff --git a/docs/planemo.engine.rst b/docs/planemo.engine.rst index 5d2130e31..a9ea05b90 100644 --- a/docs/planemo.engine.rst +++ b/docs/planemo.engine.rst @@ -4,6 +4,14 @@ planemo.engine package Submodules ---------- +planemo.engine.autoupdate module +-------------------------------- + +.. automodule:: planemo.engine.autoupdate + :members: + :undoc-members: + :show-inheritance: + planemo.engine.cwltool module ----------------------------- diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 9d6b81a42..84b5f3e7d 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -54,9 +54,6 @@ def check_conda(tool_name, ctx, **kwds): error("Conda is not installed! Try running planemo conda_init.") target = planemo.conda.conda_util.CondaTarget(tool_name) search_results = conda_util.best_search_result(target, conda_context=conda_context) - # if search_results[0]['version'] == tool_version: - # return None - # else: return search_results[0]['version'] @@ -144,7 +141,6 @@ def perform_required_update(ctx, xml_files, tool_path, requirements, tokens, xml if wrapper_version_token in v: xml_to_update[k].append({'type': 'token', 'tag': v[wrapper_version_token]['tag'], 'value': 0}) - # print(xml_to_update) # finally, update each file separately for k, v in xml_files.items(): update_xml(k, v, xml_to_update[k], wrapper_version_token, is_macro=(k != tool_path)) @@ -153,7 +149,7 @@ def perform_required_update(ctx, xml_files, tool_path, requirements, tokens, xml return set(xml_files) -def autoupdate(ctx, tool_path, modified_files=set(), **kwds): +def autoupdate_tool(ctx, tool_path, modified_files=set(), **kwds): """ Autoupdate an XML file """ diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 9e74f4d5d..8973b5275 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -68,13 +68,13 @@ def skip_requirements_option(): @options.recursive_option() @test_option() @skiplist_option() +@skip_requirements_option() @options.test_options() @options.galaxy_target_options() @options.galaxy_config_options() @options.report_level_option() @options.report_xunit() @options.fail_level_option() -@options.skip_option() @command_function def cli(ctx, paths, **kwds): """Auto-update tool requirements by checking against Conda and updating if newer versions are available.""" @@ -89,7 +89,7 @@ def cli(ctx, paths, **kwds): continue info("Auto-updating tool %s" % tool_path) try: - updated = autoupdate.autoupdate(ctx, tool_path, modified_files=modified_files, **kwds) + updated = autoupdate.autoupdate_tool(ctx, tool_path, modified_files=modified_files, **kwds) if updated: modified_files.update(updated) except Exception as e: @@ -106,14 +106,13 @@ def cli(ctx, paths, **kwds): else: with temp_directory(dir=ctx.planemo_directory) as temp_path: # only test tools in updated directories - modified_paths = [tool_path for tool_path, tool_xml in yield_tool_sources_on_paths(ctx, paths, recursive) if tool_path in modified_files] + modified_paths = [path for path, tool_xml in yield_tool_sources_on_paths(ctx, paths, recursive) if path in modified_files] info(f"Running tests for the following auto-updated tools: {', '.join(modified_paths)}") runnables = for_paths(modified_paths, temp_path=temp_path) kwds["engine"] = "galaxy" return_value = test_runnables(ctx, runnables, original_paths=paths, **kwds) - + exit_codes.append(return_value) return coalesce_return_codes(exit_codes, assert_at_least_one=assert_tools) - ctx.exit(return_value) def handle_tool_load_error(tool_path, tool_xml): From cbbbc43eff2473bd1de0abd8d3e0dd3a68654a2f Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 16 Mar 2021 17:21:45 +0100 Subject: [PATCH 39/39] update docs with @VERSION_SUFFIX@ --- docs/autoupdate.rst | 4 ++-- planemo/autoupdate.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/autoupdate.rst b/docs/autoupdate.rst index 09a5420c6..fc44ab8bd 100644 --- a/docs/autoupdate.rst +++ b/docs/autoupdate.rst @@ -25,9 +25,9 @@ Formatting tools ``autoupdate`` assumes that XML tools are formatted in a certain way - in accordance with the `IUC best practices`_. In particular: - - the tool ``version`` attribute must be set to ``@TOOL_VERSION@+galaxy0`` or ``@TOOL_VERSION@+galaxy@GALAXY_VERSION`` + - the tool ``version`` attribute must be set to ``@TOOL_VERSION@+galaxy0`` or ``@TOOL_VERSION@+galaxy@VERSION_SUFFIX@`` - A token ``@TOOL_VERSION@`` should be created which corresponds to the version number of the main requirement. - - Optionally, a token ``@GALAXY_VERSION@`` should be created, which should be an integer representing the number of times the XML wrapper has been updated since ``@TOOL_VERSION@`` was updated. + - Optionally, a token ``@VERSION_SUFFIX@`` should be created, which should be an integer representing the number of times the XML wrapper has been updated since ``@TOOL_VERSION@`` was updated. Implementing an autoupdate CI job diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 84b5f3e7d..c2964bb4d 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -77,7 +77,7 @@ def update_requirement(xml_text, tag, requirement_value): if tag_to_update['type'] == 'requirement': xml_text = update_requirement(xml_text, tag_to_update['tag'], tag_to_update['value']) if wrapper_version_token == 0 and not is_macro: - # i.e. @GALAXY_VERSION@ not specified so update the version directly in the tool tag + # i.e. @VERSION_SUFFIX@ not specified so update the version directly in the tool tag tool_tag = re.sub('version="@TOOL_VERSION@.*?"', 'version="@TOOL_VERSION@+galaxy0"', re.findall('', xml_text)[0]) xml_text = re.sub('', tool_tag, xml_text)