-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #455 from tcezard/EVA3586_split_release_in_steps
EVA-3586 - Split release job in single steps and chains them up in nextflow
- Loading branch information
Showing
15 changed files
with
724 additions
and
22 deletions.
There are no files selected for viewing
6 changes: 6 additions & 0 deletions
6
eva-accession-release-automation/publish_release_to_ftp/README_release_changelog.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
eva-accession-release-automation/publish_release_to_ftp/README_release_general_info.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| EVA RefSNP release 5 | ||
| EVA RefSNP release 6 | ||
|
|
||
| Feedback | ||
| ================================== | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
eva-accession-release-automation/run_release_in_embassy/run_release_active_for_assembly.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| # Copyright 2020 EMBL - European Bioinformatics Institute | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import click | ||
| import sys | ||
| import traceback | ||
|
|
||
| from ebi_eva_common_pyutils.logger import logging_config | ||
|
|
||
| from run_release_in_embassy.create_release_properties_file import create_release_properties_file_for_assembly | ||
| from run_release_in_embassy.release_common_utils import open_mongo_port_to_tempmongo, close_mongo_port_to_tempmongo | ||
| from ebi_eva_common_pyutils.command_utils import run_command_with_output | ||
|
|
||
|
|
||
| logger = logging_config.get_logger(__name__) | ||
|
|
||
|
|
||
| def run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, | ||
| release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, | ||
| memory): | ||
| exit_code = -1 | ||
| try: | ||
| port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, | ||
| taxonomy_id, assembly_accession, | ||
| release_species_inventory_table, | ||
| release_version) | ||
| release_properties_file = create_release_properties_file_for_assembly(private_config_xml_file, profile, | ||
| taxonomy_id, assembly_accession, | ||
| release_species_inventory_table, | ||
| release_version, assembly_release_folder, | ||
| job_name='ACTIVE_ACCESSION_RELEASE_JOB', | ||
| file_name='active_release') | ||
| release_command = 'java -Xmx{0}g -jar {1} --spring.config.location=file:{2} --spring.data.mongodb.port={3}'\ | ||
| .format(memory, release_jar_path, release_properties_file, mongo_port) | ||
| run_command_with_output("Running release pipeline for assembly: " + assembly_accession, release_command) | ||
| exit_code = 0 | ||
| except Exception as ex: | ||
| logger.error("Encountered an error while running release for assembly: " + assembly_accession + "\n" | ||
| + traceback.format_exc()) | ||
| exit_code = -1 | ||
| finally: | ||
| close_mongo_port_to_tempmongo(port_forwarding_process_id) | ||
| logger.info("Java release pipeline run completed with exit_code: " + str(exit_code)) | ||
| sys.exit(exit_code) | ||
|
|
||
|
|
||
| @click.option("--private-config-xml-file", help="ex: /path/to/eva-maven-settings.xml", required=True) | ||
| @click.option("--profile", help="Maven profile to use, ex: internal", required=True) | ||
| @click.option("--taxonomy-id", help="ex: 9913", required=True) | ||
| @click.option("--assembly-accession", help="ex: GCA_000003055.6", required=True) | ||
| @click.option("--release-species-inventory-table", default="eva_progress_tracker.clustering_release_tracker", | ||
| required=False) | ||
| @click.option("--release-version", help="ex: 2", type=int, required=True) | ||
| @click.option("--assembly-release-folder", required=True) | ||
| @click.option("--release-jar-path", required=True) | ||
| @click.option("--memory", help="Memory in GB. ex: 8", default=8, type=int, required=False) | ||
| @click.command() | ||
| def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, | ||
| release_version, assembly_release_folder, release_jar_path, memory): | ||
| logging_config.add_stdout_handler() | ||
| run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, | ||
| release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, | ||
| memory) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
78 changes: 78 additions & 0 deletions
78
...ccession-release-automation/run_release_in_embassy/run_release_deprecated_for_assembly.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| # Copyright 2020 EMBL - European Bioinformatics Institute | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import click | ||
| import sys | ||
| import traceback | ||
|
|
||
| from ebi_eva_common_pyutils.logger import logging_config | ||
|
|
||
| from run_release_in_embassy.create_release_properties_file import create_release_properties_file_for_assembly | ||
| from run_release_in_embassy.release_common_utils import open_mongo_port_to_tempmongo, close_mongo_port_to_tempmongo | ||
| from ebi_eva_common_pyutils.command_utils import run_command_with_output | ||
|
|
||
|
|
||
| logger = logging_config.get_logger(__name__) | ||
|
|
||
|
|
||
| def run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, | ||
| release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, | ||
| memory): | ||
| exit_code = -1 | ||
| try: | ||
| port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, | ||
| taxonomy_id, assembly_accession, | ||
| release_species_inventory_table, | ||
| release_version) | ||
| release_properties_file = create_release_properties_file_for_assembly(private_config_xml_file, profile, | ||
| taxonomy_id, assembly_accession, | ||
| release_species_inventory_table, | ||
| release_version, assembly_release_folder, | ||
| job_name='DEPRECATED_ACCESSION_RELEASE_JOB', | ||
| file_name='deprecated_release') | ||
| release_command = 'java -Xmx{0}g -jar {1} --spring.config.location=file:{2} --spring.data.mongodb.port={3}'\ | ||
| .format(memory, release_jar_path, release_properties_file, mongo_port) | ||
| run_command_with_output("Running release pipeline for assembly: " + assembly_accession, release_command) | ||
| exit_code = 0 | ||
| except Exception as ex: | ||
| logger.error("Encountered an error while running release for assembly: " + assembly_accession + "\n" | ||
| + traceback.format_exc()) | ||
| exit_code = -1 | ||
| finally: | ||
| close_mongo_port_to_tempmongo(port_forwarding_process_id) | ||
| logger.info("Java release pipeline run completed with exit_code: " + str(exit_code)) | ||
| sys.exit(exit_code) | ||
|
|
||
|
|
||
| @click.option("--private-config-xml-file", help="ex: /path/to/eva-maven-settings.xml", required=True) | ||
| @click.option("--profile", help="Maven profile to use, ex: internal", required=True) | ||
| @click.option("--taxonomy-id", help="ex: 9913", required=True) | ||
| @click.option("--assembly-accession", help="ex: GCA_000003055.6", required=True) | ||
| @click.option("--release-species-inventory-table", default="eva_progress_tracker.clustering_release_tracker", | ||
| required=False) | ||
| @click.option("--release-version", help="ex: 2", type=int, required=True) | ||
| @click.option("--assembly-release-folder", required=True) | ||
| @click.option("--release-jar-path", required=True) | ||
| @click.option("--memory", help="Memory in GB. ex: 8", default=8, type=int, required=False) | ||
| @click.command() | ||
| def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, | ||
| release_version, assembly_release_folder, release_jar_path, memory): | ||
| logging_config.add_stdout_handler() | ||
| run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, | ||
| release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, | ||
| memory) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.