From 3fb084eb338fb48076b1faf9daeec6f2e9e79e4b Mon Sep 17 00:00:00 2001 From: tcezard Date: Thu, 20 Jun 2024 23:51:59 +0100 Subject: [PATCH] Split release job in single steps and chains them up in nextflow --- .../create_release_properties_file.py | 7 +- .../run_release_active_for_assembly.py | 78 ++++++++++++++++++ .../run_release_deprecated_for_assembly.py | 78 ++++++++++++++++++ .../run_release_for_assembly.nf | 64 +++++++++++++-- ..._release_merged_deprecated_for_assembly.py | 78 ++++++++++++++++++ .../run_release_merged_for_assembly.py | 78 ++++++++++++++++++ .../release/configuration/BeanNames.java | 8 ++ ...ctiveAccessionReleaseJobConfiguration.java | 78 ++++++++++++++++++ ...catedAccessionReleaseJobConfiguration.java | 73 +++++++++++++++++ ...ergedAccessionReleaseJobConfiguration.java | 77 +++++++++++++++++ ...catedAccessionReleaseJobConfiguration.java | 82 +++++++++++++++++++ 11 files changed, 692 insertions(+), 9 deletions(-) create mode 100644 eva-accession-release-automation/run_release_in_embassy/run_release_active_for_assembly.py create mode 100644 eva-accession-release-automation/run_release_in_embassy/run_release_deprecated_for_assembly.py create mode 100644 eva-accession-release-automation/run_release_in_embassy/run_release_merged_deprecated_for_assembly.py create mode 100644 eva-accession-release-automation/run_release_in_embassy/run_release_merged_for_assembly.py create mode 100644 eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseJobConfiguration.java create mode 100644 eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DeprecatedAccessionReleaseJobConfiguration.java create mode 100644 eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAccessionReleaseJobConfiguration.java create mode 100644 eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedDeprecatedAccessionReleaseJobConfiguration.java diff --git a/eva-accession-release-automation/run_release_in_embassy/create_release_properties_file.py b/eva-accession-release-automation/run_release_in_embassy/create_release_properties_file.py index 0a7a298e6..7f8625d31 100644 --- a/eva-accession-release-automation/run_release_in_embassy/create_release_properties_file.py +++ b/eva-accession-release-automation/run_release_in_embassy/create_release_properties_file.py @@ -36,16 +36,17 @@ def get_release_properties_for_assembly(private_config_xml_file, profile, taxono def create_release_properties_file_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, release_version, - assembly_release_folder): + assembly_release_folder, job_name='ACCESSION_RELEASE_JOB', + file_name='release'): os.makedirs(assembly_release_folder, exist_ok=True) - output_file = "{0}/{1}_release.properties".format(assembly_release_folder, assembly_accession) + output_file = f"{assembly_release_folder}/{assembly_accession}_{file_name}.properties" release_properties = get_release_properties_for_assembly( private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, release_version ) properties_string = SpringPropertiesGenerator(profile, private_config_xml_file).get_release_properties( temp_mongo_db=release_properties['mongo_accessioning_db'], - job_name='ACCESSION_RELEASE_JOB', + job_name=job_name, assembly_accession=assembly_accession, taxonomy_accession=taxonomy_id, fasta=release_properties['fasta_path'], diff --git a/eva-accession-release-automation/run_release_in_embassy/run_release_active_for_assembly.py b/eva-accession-release-automation/run_release_in_embassy/run_release_active_for_assembly.py new file mode 100644 index 000000000..cc906ce45 --- /dev/null +++ b/eva-accession-release-automation/run_release_in_embassy/run_release_active_for_assembly.py @@ -0,0 +1,78 @@ +# Copyright 2020 EMBL - European Bioinformatics Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +import sys +import traceback + +from ebi_eva_common_pyutils.logger import logging_config + +from run_release_in_embassy.create_release_properties_file import create_release_properties_file_for_assembly +from run_release_in_embassy.release_common_utils import open_mongo_port_to_tempmongo, close_mongo_port_to_tempmongo +from ebi_eva_common_pyutils.command_utils import run_command_with_output + + +logger = logging_config.get_logger(__name__) + + +def run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, + release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, + memory): + exit_code = -1 + try: + port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, + taxonomy_id, assembly_accession, + release_species_inventory_table, + release_version) + release_properties_file = create_release_properties_file_for_assembly(private_config_xml_file, profile, + taxonomy_id, assembly_accession, + release_species_inventory_table, + release_version, assembly_release_folder, + job_name='ACTIVE_ACCESSION_RELEASE_JOB', + file_name='active_release') + release_command = 'java -Xmx{0}g -jar {1} --spring.config.location=file:{2} --spring.data.mongodb.port={3}'\ + .format(memory, release_jar_path, release_properties_file, mongo_port) + run_command_with_output("Running release pipeline for assembly: " + assembly_accession, release_command) + exit_code = 0 + except Exception as ex: + logger.error("Encountered an error while running release for assembly: " + assembly_accession + "\n" + + traceback.format_exc()) + exit_code = -1 + finally: + close_mongo_port_to_tempmongo(port_forwarding_process_id) + logger.info("Java release pipeline run completed with exit_code: " + str(exit_code)) + sys.exit(exit_code) + + +@click.option("--private-config-xml-file", help="ex: /path/to/eva-maven-settings.xml", required=True) +@click.option("--profile", help="Maven profile to use, ex: internal", required=True) +@click.option("--taxonomy-id", help="ex: 9913", required=True) +@click.option("--assembly-accession", help="ex: GCA_000003055.6", required=True) +@click.option("--release-species-inventory-table", default="eva_progress_tracker.clustering_release_tracker", + required=False) +@click.option("--release-version", help="ex: 2", type=int, required=True) +@click.option("--assembly-release-folder", required=True) +@click.option("--release-jar-path", required=True) +@click.option("--memory", help="Memory in GB. ex: 8", default=8, type=int, required=False) +@click.command() +def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, + release_version, assembly_release_folder, release_jar_path, memory): + logging_config.add_stdout_handler() + run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, + release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, + memory) + + +if __name__ == "__main__": + main() diff --git a/eva-accession-release-automation/run_release_in_embassy/run_release_deprecated_for_assembly.py b/eva-accession-release-automation/run_release_in_embassy/run_release_deprecated_for_assembly.py new file mode 100644 index 000000000..203dce30f --- /dev/null +++ b/eva-accession-release-automation/run_release_in_embassy/run_release_deprecated_for_assembly.py @@ -0,0 +1,78 @@ +# Copyright 2020 EMBL - European Bioinformatics Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +import sys +import traceback + +from ebi_eva_common_pyutils.logger import logging_config + +from run_release_in_embassy.create_release_properties_file import create_release_properties_file_for_assembly +from run_release_in_embassy.release_common_utils import open_mongo_port_to_tempmongo, close_mongo_port_to_tempmongo +from ebi_eva_common_pyutils.command_utils import run_command_with_output + + +logger = logging_config.get_logger(__name__) + + +def run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, + release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, + memory): + exit_code = -1 + try: + port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, + taxonomy_id, assembly_accession, + release_species_inventory_table, + release_version) + release_properties_file = create_release_properties_file_for_assembly(private_config_xml_file, profile, + taxonomy_id, assembly_accession, + release_species_inventory_table, + release_version, assembly_release_folder, + job_name='DEPRECATED_ACCESSION_RELEASE_JOB', + file_name='deprecated_release') + release_command = 'java -Xmx{0}g -jar {1} --spring.config.location=file:{2} --spring.data.mongodb.port={3}'\ + .format(memory, release_jar_path, release_properties_file, mongo_port) + run_command_with_output("Running release pipeline for assembly: " + assembly_accession, release_command) + exit_code = 0 + except Exception as ex: + logger.error("Encountered an error while running release for assembly: " + assembly_accession + "\n" + + traceback.format_exc()) + exit_code = -1 + finally: + close_mongo_port_to_tempmongo(port_forwarding_process_id) + logger.info("Java release pipeline run completed with exit_code: " + str(exit_code)) + sys.exit(exit_code) + + +@click.option("--private-config-xml-file", help="ex: /path/to/eva-maven-settings.xml", required=True) +@click.option("--profile", help="Maven profile to use, ex: internal", required=True) +@click.option("--taxonomy-id", help="ex: 9913", required=True) +@click.option("--assembly-accession", help="ex: GCA_000003055.6", required=True) +@click.option("--release-species-inventory-table", default="eva_progress_tracker.clustering_release_tracker", + required=False) +@click.option("--release-version", help="ex: 2", type=int, required=True) +@click.option("--assembly-release-folder", required=True) +@click.option("--release-jar-path", required=True) +@click.option("--memory", help="Memory in GB. ex: 8", default=8, type=int, required=False) +@click.command() +def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, + release_version, assembly_release_folder, release_jar_path, memory): + logging_config.add_stdout_handler() + run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, + release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, + memory) + + +if __name__ == "__main__": + main() diff --git a/eva-accession-release-automation/run_release_in_embassy/run_release_for_assembly.nf b/eva-accession-release-automation/run_release_in_embassy/run_release_for_assembly.nf index 7f71d5d5e..8f5a7cedb 100644 --- a/eva-accession-release-automation/run_release_in_embassy/run_release_for_assembly.nf +++ b/eva-accession-release-automation/run_release_in_embassy/run_release_for_assembly.nf @@ -3,10 +3,11 @@ nextflow.enable.dsl=2 workflow { - initiate_release_status_for_assembly('initiate') | copy_accessioning_collections_to_embassy | run_release_for_assembly | \ - merge_dbsnp_eva_release_files | sort_bgzip_index_release_files | validate_release_vcf_files | \ - analyze_vcf_validation_results | count_rs_ids_in_release_files | validate_rs_release_files | \ - update_sequence_names_to_ena | update_release_status_for_assembly + initiate_release_status_for_assembly('initiate') | copy_accessioning_collections_to_embassy | \ + run_release_active_for_assembly | run_release_merged_for_assembly | run_release_deprecated_for_assembly | \ + run_release_merged_deprecated_for_assembly | merge_dbsnp_eva_release_files | sort_bgzip_index_release_files | \ + validate_release_vcf_files | analyze_vcf_validation_results | count_rs_ids_in_release_files | \ + validate_rs_release_files | update_sequence_names_to_ena | update_release_status_for_assembly } process initiate_release_status_for_assembly { @@ -43,7 +44,7 @@ process copy_accessioning_collections_to_embassy { """ } -process run_release_for_assembly { +process run_release_active_for_assembly { label 'long_time', 'med_mem' @@ -56,7 +57,58 @@ process run_release_for_assembly { script: """ export PYTHONPATH=$params.python_path - $params.executable.python_interpreter -m run_release_in_embassy.run_release_for_assembly --private-config-xml-file $params.maven.settings_file --profile $params.maven.environment --taxonomy-id $params.taxonomy --assembly-accession $params.assembly --release-species-inventory-table eva_progress_tracker.clustering_release_tracker --release-version $params.release_version --assembly-release-folder $params.assembly_folder --release-jar-path $params.jar.release_pipeline 1>> $params.log_file 2>&1 + $params.executable.python_interpreter -m run_release_in_embassy.run_release_active_for_assembly --private-config-xml-file $params.maven.settings_file --profile $params.maven.environment --taxonomy-id $params.taxonomy --assembly-accession $params.assembly --release-species-inventory-table eva_progress_tracker.clustering_release_tracker --release-version $params.release_version --assembly-release-folder $params.assembly_folder --release-jar-path $params.jar.release_pipeline 1>> $params.log_file 2>&1 + """ +} + +process run_release_merged_for_assembly { + + label 'long_time', 'med_mem' + + input: + val flag + + output: + val true, emit: flag + + script: + """ + export PYTHONPATH=$params.python_path + $params.executable.python_interpreter -m run_release_in_embassy.run_release_merged_for_assembly --private-config-xml-file $params.maven.settings_file --profile $params.maven.environment --taxonomy-id $params.taxonomy --assembly-accession $params.assembly --release-species-inventory-table eva_progress_tracker.clustering_release_tracker --release-version $params.release_version --assembly-release-folder $params.assembly_folder --release-jar-path $params.jar.release_pipeline 1>> $params.log_file 2>&1 + """ +} + +process run_release_deprecated_for_assembly { + + label 'long_time', 'med_mem' + + input: + val flag + + output: + val true, emit: flag + + script: + """ + export PYTHONPATH=$params.python_path + $params.executable.python_interpreter -m run_release_in_embassy.run_release_deprecated_for_assembly --private-config-xml-file $params.maven.settings_file --profile $params.maven.environment --taxonomy-id $params.taxonomy --assembly-accession $params.assembly --release-species-inventory-table eva_progress_tracker.clustering_release_tracker --release-version $params.release_version --assembly-release-folder $params.assembly_folder --release-jar-path $params.jar.release_pipeline 1>> $params.log_file 2>&1 + """ +} + +process run_release_merged_deprecated_for_assembly { + + label 'long_time', 'med_mem' + + input: + val flag + + output: + val true, emit: flag + + script: + """ + export PYTHONPATH=$params.python_path + $params.executable.python_interpreter -m run_release_in_embassy.run_release_merged_deprecated_for_assembly --private-config-xml-file $params.maven.settings_file --profile $params.maven.environment --taxonomy-id $params.taxonomy --assembly-accession $params.assembly --release-species-inventory-table eva_progress_tracker.clustering_release_tracker --release-version $params.release_version --assembly-release-folder $params.assembly_folder --release-jar-path $params.jar.release_pipeline 1>> $params.log_file 2>&1 """ } diff --git a/eva-accession-release-automation/run_release_in_embassy/run_release_merged_deprecated_for_assembly.py b/eva-accession-release-automation/run_release_in_embassy/run_release_merged_deprecated_for_assembly.py new file mode 100644 index 000000000..1dc6f9069 --- /dev/null +++ b/eva-accession-release-automation/run_release_in_embassy/run_release_merged_deprecated_for_assembly.py @@ -0,0 +1,78 @@ +# Copyright 2020 EMBL - European Bioinformatics Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +import sys +import traceback + +from ebi_eva_common_pyutils.logger import logging_config + +from run_release_in_embassy.create_release_properties_file import create_release_properties_file_for_assembly +from run_release_in_embassy.release_common_utils import open_mongo_port_to_tempmongo, close_mongo_port_to_tempmongo +from ebi_eva_common_pyutils.command_utils import run_command_with_output + + +logger = logging_config.get_logger(__name__) + + +def run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, + release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, + memory): + exit_code = -1 + try: + port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, + taxonomy_id, assembly_accession, + release_species_inventory_table, + release_version) + release_properties_file = create_release_properties_file_for_assembly(private_config_xml_file, profile, + taxonomy_id, assembly_accession, + release_species_inventory_table, + release_version, assembly_release_folder, + job_name='MERGED_DEPRECATED_ACCESSION_RELEASE_JOB', + file_name='merged_deprecated_release') + release_command = 'java -Xmx{0}g -jar {1} --spring.config.location=file:{2} --spring.data.mongodb.port={3}'\ + .format(memory, release_jar_path, release_properties_file, mongo_port) + run_command_with_output("Running release pipeline for assembly: " + assembly_accession, release_command) + exit_code = 0 + except Exception as ex: + logger.error("Encountered an error while running release for assembly: " + assembly_accession + "\n" + + traceback.format_exc()) + exit_code = -1 + finally: + close_mongo_port_to_tempmongo(port_forwarding_process_id) + logger.info("Java release pipeline run completed with exit_code: " + str(exit_code)) + sys.exit(exit_code) + + +@click.option("--private-config-xml-file", help="ex: /path/to/eva-maven-settings.xml", required=True) +@click.option("--profile", help="Maven profile to use, ex: internal", required=True) +@click.option("--taxonomy-id", help="ex: 9913", required=True) +@click.option("--assembly-accession", help="ex: GCA_000003055.6", required=True) +@click.option("--release-species-inventory-table", default="eva_progress_tracker.clustering_release_tracker", + required=False) +@click.option("--release-version", help="ex: 2", type=int, required=True) +@click.option("--assembly-release-folder", required=True) +@click.option("--release-jar-path", required=True) +@click.option("--memory", help="Memory in GB. ex: 8", default=8, type=int, required=False) +@click.command() +def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, + release_version, assembly_release_folder, release_jar_path, memory): + logging_config.add_stdout_handler() + run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, + release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, + memory) + + +if __name__ == "__main__": + main() diff --git a/eva-accession-release-automation/run_release_in_embassy/run_release_merged_for_assembly.py b/eva-accession-release-automation/run_release_in_embassy/run_release_merged_for_assembly.py new file mode 100644 index 000000000..4ae717074 --- /dev/null +++ b/eva-accession-release-automation/run_release_in_embassy/run_release_merged_for_assembly.py @@ -0,0 +1,78 @@ +# Copyright 2020 EMBL - European Bioinformatics Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +import sys +import traceback + +from ebi_eva_common_pyutils.logger import logging_config + +from run_release_in_embassy.create_release_properties_file import create_release_properties_file_for_assembly +from run_release_in_embassy.release_common_utils import open_mongo_port_to_tempmongo, close_mongo_port_to_tempmongo +from ebi_eva_common_pyutils.command_utils import run_command_with_output + + +logger = logging_config.get_logger(__name__) + + +def run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, + release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, + memory): + exit_code = -1 + try: + port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, + taxonomy_id, assembly_accession, + release_species_inventory_table, + release_version) + release_properties_file = create_release_properties_file_for_assembly(private_config_xml_file, profile, + taxonomy_id, assembly_accession, + release_species_inventory_table, + release_version, assembly_release_folder, + job_name='MERGED_ACCESSION_RELEASE_JOB', + file_name='merged_release') + release_command = 'java -Xmx{0}g -jar {1} --spring.config.location=file:{2} --spring.data.mongodb.port={3}'\ + .format(memory, release_jar_path, release_properties_file, mongo_port) + run_command_with_output("Running release pipeline for assembly: " + assembly_accession, release_command) + exit_code = 0 + except Exception as ex: + logger.error("Encountered an error while running release for assembly: " + assembly_accession + "\n" + + traceback.format_exc()) + exit_code = -1 + finally: + close_mongo_port_to_tempmongo(port_forwarding_process_id) + logger.info("Java release pipeline run completed with exit_code: " + str(exit_code)) + sys.exit(exit_code) + + +@click.option("--private-config-xml-file", help="ex: /path/to/eva-maven-settings.xml", required=True) +@click.option("--profile", help="Maven profile to use, ex: internal", required=True) +@click.option("--taxonomy-id", help="ex: 9913", required=True) +@click.option("--assembly-accession", help="ex: GCA_000003055.6", required=True) +@click.option("--release-species-inventory-table", default="eva_progress_tracker.clustering_release_tracker", + required=False) +@click.option("--release-version", help="ex: 2", type=int, required=True) +@click.option("--assembly-release-folder", required=True) +@click.option("--release-jar-path", required=True) +@click.option("--memory", help="Memory in GB. ex: 8", default=8, type=int, required=False) +@click.command() +def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, + release_version, assembly_release_folder, release_jar_path, memory): + logging_config.add_stdout_handler() + run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession, + release_species_inventory_table, release_version, assembly_release_folder, release_jar_path, + memory) + + +if __name__ == "__main__": + main() diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/BeanNames.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/BeanNames.java index 112cbec18..ffc1fbfb7 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/BeanNames.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/BeanNames.java @@ -18,6 +18,14 @@ public class BeanNames { // common beans + public static final String ACTIVE_ACCESSION_RELEASE_JOB = "ACTIVE_ACCESSION_RELEASE_JOB"; + + public static final String MERGED_ACCESSION_RELEASE_JOB = "MERGED_ACCESSION_RELEASE_JOB"; + + public static final String DEPRECATED_ACCESSION_RELEASE_JOB = "DEPRECATED_ACCESSION_RELEASE_JOB"; + + public static final String MERGED_DEPRECATED_ACCESSION_RELEASE_JOB = "MERGED_DEPRECATED_ACCESSION_RELEASE_JOB"; + public static final String ACCESSION_RELEASE_JOB = "ACCESSION_RELEASE_JOB"; public static final String CREATE_INCREMENTAL_ACCESSION_RELEASE_JOB = "CREATE_INCREMENTAL_ACCESSION_RELEASE_JOB"; diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseJobConfiguration.java new file mode 100644 index 000000000..3086611db --- /dev/null +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseJobConfiguration.java @@ -0,0 +1,78 @@ +/* + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.ac.ebi.eva.accession.release.configuration.batch.jobs; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.FlowJobBuilder; +import org.springframework.batch.core.job.flow.Flow; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateDeprecatedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMergedDeprecatedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMergedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMultimapReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.ListContigsStepConfiguration; + +import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.*; + +@Configuration +@EnableBatchProcessing +@Import({ListContigsStepConfiguration.class, + CreateReleaseStepConfiguration.class}) +public class ActiveAccessionReleaseJobConfiguration { + + + @Bean(ACTIVE_ACCESSION_RELEASE_JOB) + public Job ActiveAccessionReleaseJob(JobBuilderFactory jobBuilderFactory, + Flow dbsnpFlow, + Flow evaFlow) { + FlowBuilder flowBuilder = jobBuilderFactory.get(ACTIVE_ACCESSION_RELEASE_JOB) + .incrementer(new RunIdIncrementer()) + .start(dbsnpFlow) + .next(evaFlow); + FlowJobBuilder jobBuilder = flowBuilder.build(); + return jobBuilder.build(); + } + + @Bean + public Flow dbsnpFlow( + @Qualifier(LIST_DBSNP_ACTIVE_CONTIGS_STEP) Step listActiveContigsStep, + @Qualifier(RELEASE_DBSNP_MAPPED_ACTIVE_VARIANTS_STEP) Step createReleaseStep) { + return new FlowBuilder(DBSNP_FLOW) + .start(listActiveContigsStep) + .next(createReleaseStep) + .build(); + } + + @Bean + public Flow evaFlow( + @Qualifier(LIST_EVA_ACTIVE_CONTIGS_STEP) Step listActiveContigsStep, + @Qualifier(RELEASE_EVA_MAPPED_ACTIVE_VARIANTS_STEP) Step createReleaseStep) { + return new FlowBuilder(EVA_FLOW) + .start(listActiveContigsStep) + .next(createReleaseStep) + .build(); + } +} diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DeprecatedAccessionReleaseJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DeprecatedAccessionReleaseJobConfiguration.java new file mode 100644 index 000000000..28605b0cb --- /dev/null +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DeprecatedAccessionReleaseJobConfiguration.java @@ -0,0 +1,73 @@ +/* + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.ac.ebi.eva.accession.release.configuration.batch.jobs; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.FlowJobBuilder; +import org.springframework.batch.core.job.flow.Flow; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateDeprecatedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMergedDeprecatedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMergedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMultimapReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.ListContigsStepConfiguration; + +import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.*; + +@Configuration +@EnableBatchProcessing +@Import({CreateDeprecatedReleaseStepConfiguration.class}) +public class DeprecatedAccessionReleaseJobConfiguration { + + /** + */ + @Bean(DEPRECATED_ACCESSION_RELEASE_JOB) + public Job accessionReleaseJob(JobBuilderFactory jobBuilderFactory, + Flow dbsnpFlow, + Flow evaFlow) { + FlowBuilder flowBuilder = jobBuilderFactory.get(DEPRECATED_ACCESSION_RELEASE_JOB) + .incrementer(new RunIdIncrementer()) + .start(dbsnpFlow) + .next(evaFlow); + FlowJobBuilder jobBuilder = flowBuilder.build(); + return jobBuilder.build(); + } + + @Bean + public Flow dbsnpFlow( + @Qualifier(RELEASE_DBSNP_MAPPED_DEPRECATED_VARIANTS_STEP) Step createDeprecatedReleaseStep) { + return new FlowBuilder(DBSNP_FLOW) + .start(createDeprecatedReleaseStep) + .build(); + } + + @Bean + public Flow evaFlow(@Qualifier(RELEASE_EVA_MAPPED_DEPRECATED_VARIANTS_STEP) Step createDeprecatedReleaseStep) { + return new FlowBuilder(EVA_FLOW) + .start(createDeprecatedReleaseStep) + .build(); + } +} diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAccessionReleaseJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAccessionReleaseJobConfiguration.java new file mode 100644 index 000000000..10e927075 --- /dev/null +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAccessionReleaseJobConfiguration.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.ac.ebi.eva.accession.release.configuration.batch.jobs; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.FlowJobBuilder; +import org.springframework.batch.core.job.flow.Flow; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateDeprecatedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMergedDeprecatedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMergedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMultimapReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.ListContigsStepConfiguration; + +import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.*; + +@Configuration +@EnableBatchProcessing +@Import({ListContigsStepConfiguration.class, + CreateMergedReleaseStepConfiguration.class}) +public class MergedAccessionReleaseJobConfiguration { + + @Bean(MERGED_ACCESSION_RELEASE_JOB) + public Job accessionReleaseJob(JobBuilderFactory jobBuilderFactory, + Flow dbsnpFlow, + Flow evaFlow) { + FlowBuilder flowBuilder = jobBuilderFactory.get(MERGED_ACCESSION_RELEASE_JOB) + .incrementer(new RunIdIncrementer()) + .start(dbsnpFlow) + .next(evaFlow); + FlowJobBuilder jobBuilder = flowBuilder.build(); + return jobBuilder.build(); + } + + @Bean + public Flow dbsnpFlow( + @Qualifier(LIST_DBSNP_MERGED_CONTIGS_STEP) Step listMergedContigsStep, + @Qualifier(RELEASE_DBSNP_MAPPED_MERGED_VARIANTS_STEP) Step createMergedReleaseStep) { + return new FlowBuilder(DBSNP_FLOW) + .start(listMergedContigsStep) + .next(createMergedReleaseStep) + .build(); + } + + @Bean + public Flow evaFlow( + @Qualifier(LIST_EVA_MERGED_CONTIGS_STEP) Step listMergedContigsStep, + @Qualifier(RELEASE_EVA_MAPPED_MERGED_VARIANTS_STEP) Step createMergedReleaseStep) { + return new FlowBuilder(EVA_FLOW) + .start(listMergedContigsStep) + .next(createMergedReleaseStep) + .build(); + } +} diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedDeprecatedAccessionReleaseJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedDeprecatedAccessionReleaseJobConfiguration.java new file mode 100644 index 000000000..8a1266bbc --- /dev/null +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedDeprecatedAccessionReleaseJobConfiguration.java @@ -0,0 +1,82 @@ +/* + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.ac.ebi.eva.accession.release.configuration.batch.jobs; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.FlowJobBuilder; +import org.springframework.batch.core.job.flow.Flow; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateDeprecatedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMergedDeprecatedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMergedReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateMultimapReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.CreateReleaseStepConfiguration; +import uk.ac.ebi.eva.accession.release.configuration.batch.steps.ListContigsStepConfiguration; + +import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.*; + +@Configuration +@EnableBatchProcessing +@Import({ListContigsStepConfiguration.class, + CreateReleaseStepConfiguration.class, + CreateDeprecatedReleaseStepConfiguration.class, + CreateMergedDeprecatedReleaseStepConfiguration.class, + CreateMergedReleaseStepConfiguration.class, + CreateMultimapReleaseStepConfiguration.class}) +public class MergedDeprecatedAccessionReleaseJobConfiguration { + + /** + * Note that we are just concatenating the DBSNP and EVA steps one after the other. We could configure them to run + * in parallel if we want to. See ParallelStatisticsAndAnnotationFlowConfiguration in eva-pipeline for an example + * of asynchronous parallel flows. + */ + @Bean(MERGED_DEPRECATED_ACCESSION_RELEASE_JOB) + public Job accessionReleaseJob(JobBuilderFactory jobBuilderFactory, + Flow dbsnpFlow, + Flow evaFlow) { + FlowBuilder flowBuilder = jobBuilderFactory.get(MERGED_DEPRECATED_ACCESSION_RELEASE_JOB) + .incrementer(new RunIdIncrementer()) + .start(dbsnpFlow) + .next(evaFlow); + FlowJobBuilder jobBuilder = flowBuilder.build(); + return jobBuilder.build(); + } + + @Bean + public Flow dbsnpFlow( + @Qualifier(RELEASE_DBSNP_MAPPED_MERGED_DEPRECATED_VARIANTS_STEP) Step createMergedDeprecatedReleaseStep) { + return new FlowBuilder(DBSNP_FLOW) + .start(createMergedDeprecatedReleaseStep) + .build(); + } + + @Bean + public Flow evaFlow( + @Qualifier(RELEASE_EVA_MAPPED_MERGED_DEPRECATED_VARIANTS_STEP) Step createMergedDeprecatedReleaseStep) { + return new FlowBuilder(EVA_FLOW) + .start(createMergedDeprecatedReleaseStep) + .build(); + } +}