From b8a71ecc984f0abefeea72662d5a38518663267b Mon Sep 17 00:00:00 2001 From: dglemos Date: Wed, 15 Feb 2023 17:03:26 +0000 Subject: [PATCH 01/13] Add slurm --- .../Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm index 5f5cc0f48..3c551dcd4 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm @@ -85,8 +85,8 @@ sub resource_classes { my ($self) = @_; return { %{$self->SUPER::resource_classes}, - '8Gb_8c_job' => {'LSF' => '-n 8 -q production -R"select[mem>8000] rusage[mem=8000]" -M8000' }, - '4Gb_job' => {'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000'}, + '8Gb_8c_job' => { SLURM => "--cpus-per-task=8 --partition=standard --time=4:00:00 --mem=8G" }, + '4Gb_job' => { SLURM => "--partition=standard --time=4:00:00 --mem=8G" }, }; } From a6c14d7e36a131c7e4fe252cca035f18722a8f8c Mon Sep 17 00:00:00 2001 From: dglemos Date: Thu, 16 Feb 2023 10:44:50 +0000 Subject: [PATCH 02/13] Define default --- .../Variation/Pipeline/SpliceAI/SpliceAI_conf.pm | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm index 3c551dcd4..cab131719 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm @@ -54,7 +54,7 @@ sub default_options { pipeline_name => 'spliceai_scores', main_dir => $self->o('main_dir'), # main directory where all files and directories are going to be stored input_directory => $self->o('main_dir') . '/input_vcf_files', # input files - split_vcf_no_header_dir => $self->o('main_dir') . '/split_vcf_no_header', # contains the input files after being splitted (files without headers) + split_vcf_no_header_dir => $self->o('main_dir') . '/split_vcf_no_header', # contains the input files after being splitted (files without headers) split_vcf_input_dir => $self->o('main_dir') . '/split_vcf_input', # contains the splitted input vcf files with headers, these are the files used to run SpliceAI split_vcf_output_dir => $self->o('main_dir') . '/split_vcf_output', # temporary output files, still splitted output_dir => $self->o('main_dir') . '/output', # final output files already merged by chromosome @@ -63,8 +63,8 @@ sub default_options { step_size => 4000, # number of variants used to split the main vcf files check_transcripts => 0, # if set to 1 checks which are the new MANE Select transcripts for the last months and only calculates SpliceAI scores for these variants overlapping these transcripts transcripts_from_file => undef, - time_interval => 4, # checks which transcripts were updated/created in the last 4 months; only used if check_transcripts = 1 and we want to check the new transcripts in the core db - masked_scores => 1, # calculate masked scores + time_interval => 4, # checks which transcripts were updated/created in the last 4 months; only used if check_transcripts = 1 and we want to check the new transcripts in the core db + masked_scores => 1, # calculate masked scores registry => undef, # database where new MANE transcripts are going to be checked; only used if check_transcripts = 1 output_file_name => 'spliceai_final_scores_', @@ -85,8 +85,9 @@ sub resource_classes { my ($self) = @_; return { %{$self->SUPER::resource_classes}, - '8Gb_8c_job' => { SLURM => "--cpus-per-task=8 --partition=standard --time=4:00:00 --mem=8G" }, - '4Gb_job' => { SLURM => "--partition=standard --time=4:00:00 --mem=8G" }, + '8Gb_8c_job' => { 'SLURM' => "--cpus-per-task=8 --partition=standard --time=4:00:00 --mem=8G" }, + '4Gb_job' => { 'SLURM' => "--partition=standard --time=4:00:00 --mem=4G" }, + 'default' => { 'SLURM' => "--partition=standard --time=1:00:00 --mem=1G" }, }; } From abc603957745ab1e335050af623b6298314aa791 Mon Sep 17 00:00:00 2001 From: dglemos Date: Mon, 6 Mar 2023 14:06:11 +0000 Subject: [PATCH 03/13] Update spliceai resource class --- .../Pipeline/SpliceAI/SpliceAI_conf.pm | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm index cab131719..fe2c3d09d 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm @@ -83,11 +83,23 @@ sub default_options { sub resource_classes { my ($self) = @_; + + my $step_size = $self->o('step_size'); + my $time = 4; + if($step_size > 4000) { + $time = $step_size/1000; + } + return { %{$self->SUPER::resource_classes}, - '8Gb_8c_job' => { 'SLURM' => "--cpus-per-task=8 --partition=standard --time=4:00:00 --mem=8G" }, - '4Gb_job' => { 'SLURM' => "--partition=standard --time=4:00:00 --mem=4G" }, - 'default' => { 'SLURM' => "--partition=standard --time=1:00:00 --mem=1G" }, + '8Gb_8c_job' => { 'LSF' => '-n 8 -q production -R"select[mem>8000] rusage[mem=8000]" -M8000', + 'SLURM' => "--cpus-per-task=8 --partition=standard --time=$time:00:00 --mem=8G" + }, + '4Gb_job' => { 'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000', + 'SLURM' => "--partition=standard --time=$time:00:00 --mem=4G" + }, + 'default' => { 'LSF' => '-R"select[mem>2000] rusage[mem=2000]" -M2000', + 'SLURM' => "--partition=standard --time=1:00:00 --mem=1G" } }; } From 49e1e4cf9f4d3df553aaf5e74d85c40fc984e79e Mon Sep 17 00:00:00 2001 From: dglemos Date: Mon, 6 Mar 2023 14:07:43 +0000 Subject: [PATCH 04/13] Request 1G of memory --- .../Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm index fe2c3d09d..45b3694f0 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm @@ -98,7 +98,7 @@ sub resource_classes { '4Gb_job' => { 'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000', 'SLURM' => "--partition=standard --time=$time:00:00 --mem=4G" }, - 'default' => { 'LSF' => '-R"select[mem>2000] rusage[mem=2000]" -M2000', + 'default' => { 'LSF' => '-R"select[mem>1000] rusage[mem=1000]" -M1000', 'SLURM' => "--partition=standard --time=1:00:00 --mem=1G" } }; } From d17bf86826f99191b38bdd599f70c13edcb79ecd Mon Sep 17 00:00:00 2001 From: dglemos Date: Mon, 6 Mar 2023 14:27:17 +0000 Subject: [PATCH 05/13] Add slurm --- .../PhenotypeAnnotation_conf.pm | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/PhenotypeAnnotation/PhenotypeAnnotation_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/PhenotypeAnnotation/PhenotypeAnnotation_conf.pm index d856b1ccb..747be5427 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/PhenotypeAnnotation/PhenotypeAnnotation_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/PhenotypeAnnotation/PhenotypeAnnotation_conf.pm @@ -143,9 +143,13 @@ sub default_options { default_lsf_options => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000', medmem_lsf_options => '-qproduction -R"select[mem>4000] rusage[mem=4000]" -M4000', urgent_lsf_options => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000', - highmem_lsf_options => '-qproduction -R"select[mem>15000] rusage[mem=15000] span[hosts=1]" -M15000 -n4', # this is LSF speak for "give me 15GB of memory" + highmem_lsf_options => '-qproduction -R"select[mem>15000] rusage[mem=15000] span[hosts=1]" -M15000 -n4', long_lsf_options => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000', + default_slurm_options => '--partition=standard --time=12:00:00 --mem=2G', + highmem_slurm_options => '--partition=standard --time=12:00:00 --mem=15G', + long_slurm_options => '--partition=standard --time=24:00:00 --mem=2G', + # connection parameters for the hive database, you should supply the hive_db_password # option on the command line to init_pipeline.pl (parameters for the target database # should be set in the registry file defined above) @@ -176,10 +180,13 @@ sub default_options { sub resource_classes { my ($self) = @_; return { - 'default' => { 'LSF' => $self->o('default_lsf_options') }, + 'default' => { 'LSF' => $self->o('default_lsf_options'), + 'SLURM' => $self->o('default_slurm_options') }, 'urgent' => { 'LSF' => $self->o('urgent_lsf_options') }, - 'highmem' => { 'LSF' => $self->o('highmem_lsf_options') }, - 'long' => { 'LSF' => $self->o('long_lsf_options') }, + 'highmem' => { 'LSF' => $self->o('highmem_lsf_options'), + 'SLURM' => $self->o('highmem_slurm_options') }, + 'long' => { 'LSF' => $self->o('long_lsf_options'), + 'SLURM' => $self->o('long_slurm_options') }, 'medmem' => { 'LSF' => $self->o('medmem_lsf_options') }, }; } @@ -210,7 +217,7 @@ sub pipeline_analyses { '5' => [ 'import_rgd' ], '6' => [ 'import_zfin' ], - '14' => [ 'import_wormbase' ], + '14' => [ 'import_wormbase' ], }, }, @@ -242,7 +249,7 @@ sub pipeline_analyses { }, -input_ids => [], #default -hive_capacity => 1, - -rc_name => 'default', + -rc_name => 'long', -flow_into => { 2 => [ 'check_gwas'] }, @@ -582,18 +589,18 @@ sub pipeline_analyses { }, }, { -logic_name => 'import_wormbase', - -module => 'Bio::EnsEMBL::Variation::Pipeline::PhenotypeAnnotation::ImportWORMBASE', - -parameters => { - source_version => $self->o('wormbase_release'), + -module => 'Bio::EnsEMBL::Variation::Pipeline::PhenotypeAnnotation::ImportWORMBASE', + -parameters => { + source_version => $self->o('wormbase_release'), @common_params, }, - -input_ids => [], - -hive_capacity => 1, - -rc_name => 'default', - -flow_into => { - 1 => [ 'check_phenotypes'] - }, - }, + -input_ids => [], + -hive_capacity => 1, + -rc_name => 'default', + -flow_into => { + 1 => [ 'check_phenotypes'] + }, + }, { -logic_name => 'import_ontology_mapping', -module => 'Bio::EnsEMBL::Variation::Pipeline::PhenotypeAnnotation::OntologyMapping', -parameters => { From 212ff0e413859e8d0d7c0e282595ca6145cab3d1 Mon Sep 17 00:00:00 2001 From: dglemos Date: Mon, 6 Mar 2023 15:04:20 +0000 Subject: [PATCH 06/13] Update regulation pipeline for slurm --- .../Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm index a594e8777..ec8973272 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm @@ -88,8 +88,10 @@ sub resource_classes { # requirements, queue parameters etc.) to suit your own data return { - 'default' => { 'LSF' => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000'}, - 'highmem' => { 'LSF' => '-qproduction -R"select[mem>15000] rusage[mem=15000]" -M15000'}, + 'default' => { 'LSF' => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000', + 'SLURM' => "--partition=standard --time=4:00:00 --mem=2G" }, + 'highmem' => { 'LSF' => '-qproduction -R"select[mem>15000] rusage[mem=15000]" -M15000', + 'SLURM' => "--partition=standard --time=28:00:00 --mem=15G"}, }; } From 2f3064f43bc0cd4b97283789e0f767a03f417c12 Mon Sep 17 00:00:00 2001 From: dglemos Date: Mon, 6 Mar 2023 17:36:04 +0000 Subject: [PATCH 07/13] Add long resource --- modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm index ec8973272..7ad806821 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm @@ -92,6 +92,8 @@ sub resource_classes { 'SLURM' => "--partition=standard --time=4:00:00 --mem=2G" }, 'highmem' => { 'LSF' => '-qproduction -R"select[mem>15000] rusage[mem=15000]" -M15000', 'SLURM' => "--partition=standard --time=28:00:00 --mem=15G"}, + 'long' => { 'LSF' => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000', + 'SLURM' => "--partition=standard --time=28:00:00 --mem=2G"}, }; } From b0536e47115636848ca93bb66a6e8bce86ed43bb Mon Sep 17 00:00:00 2001 From: dglemos Date: Mon, 6 Mar 2023 17:37:53 +0000 Subject: [PATCH 08/13] Add slurm to variation consequence --- .../Pipeline/VariationConsequence_conf.pm | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm index 672663134..c9aff9182 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm @@ -96,7 +96,7 @@ sub default_options { disambiguate_single_nucleotide_alleles => 0, # shifting variants within repeated regions in the 3' direction is switched off by default. - + prevent_shifting => 1, # configuration for the various resource options used in the pipeline @@ -109,11 +109,18 @@ sub default_options { medmem_lsf_options => '-qproduction -R"select[mem>5000] rusage[mem=5000]" -M5000', highmem_lsf_options => '-qproduction -R"select[mem>15000] rusage[mem=15000] span[hosts=1]" -M15000 -n4', # this is LSF speak for "give me 15GB of memory" + default_slurm_options => '--partition=standard --time=24:00:00 --mem=2G', + default_long_slurm_options => '--partition=standard --time=140:00:00 --mem=2G', + medmem_slurm_options => '--partition=standard --time=24:00:00 --mem=5G', + medmem_long_slurm_options => '--partition=standard --time=140:00:00 --mem=5G', + highmem_slurm_options => '--partition=standard --time=24:00:00 --mem=15G', + highmem_long_slurm_options => '--partition=standard --time=140:00:00 --mem=15G', + # options controlling the number of workers used for the parallelisable analyses # these default values seem to work for most species set_variation_class_capacity => 10, - + # set this flag to 1 to include LRG transcripts in the transcript effect analysis include_lrg => 1, @@ -150,7 +157,7 @@ sub default_options { # Human runs switch off run_var_class and set max_distance to 0 by default. To override # this behaviour, set this flag to 1 - human_default_override => 0, + human_default_override => 0, # connection parameters for the hive database, you should supply the hive_db_password # option on the command line to init_pipeline.pl (parameters for the target database @@ -173,7 +180,7 @@ sub default_options { -host => $self->o('hive_db_host'), -port => $self->o('hive_db_port'), -user => $self->o('hive_db_user'), - -pass => $self->o('hive_db_password'), + -pass => $self->o('hive_db_password'), -dbname => $self->o('hive_db_name'), -driver => 'mysql', -reconnect_when_lost => 1 @@ -193,9 +200,12 @@ sub pipeline_wide_parameters { # these parameter values are visible to all anal sub resource_classes { my ($self) = @_; return { - 'default' => { 'LSF' => $self->o('default_lsf_options') }, - 'highmem' => { 'LSF' => $self->o('highmem_lsf_options') }, - 'medmem' => { 'LSF' => $self->o('medmem_lsf_options') }, + 'default' => { 'LSF' => $self->o('default_lsf_options'), + 'SLURM' => $self->o('default_slurm_options') }, + 'highmem' => { 'LSF' => $self->o('highmem_lsf_options'), + 'SLURM' => $self->o('highmem_slurm_options') }, + 'medmem' => { 'LSF' => $self->o('medmem_lsf_options'), + 'SLURM' => $self->o('medmem_slurm_options') }, }; } @@ -432,4 +442,3 @@ sub pipeline_analyses { } 1; - From 4ceb691701225158448bdad58a878943933a8e88 Mon Sep 17 00:00:00 2001 From: dglemos Date: Fri, 19 May 2023 14:05:04 +0100 Subject: [PATCH 09/13] Revert SpliceAI update --- .../Pipeline/SpliceAI/SpliceAI_conf.pm | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm index 45b3694f0..5f5cc0f48 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/SpliceAI/SpliceAI_conf.pm @@ -54,7 +54,7 @@ sub default_options { pipeline_name => 'spliceai_scores', main_dir => $self->o('main_dir'), # main directory where all files and directories are going to be stored input_directory => $self->o('main_dir') . '/input_vcf_files', # input files - split_vcf_no_header_dir => $self->o('main_dir') . '/split_vcf_no_header', # contains the input files after being splitted (files without headers) + split_vcf_no_header_dir => $self->o('main_dir') . '/split_vcf_no_header', # contains the input files after being splitted (files without headers) split_vcf_input_dir => $self->o('main_dir') . '/split_vcf_input', # contains the splitted input vcf files with headers, these are the files used to run SpliceAI split_vcf_output_dir => $self->o('main_dir') . '/split_vcf_output', # temporary output files, still splitted output_dir => $self->o('main_dir') . '/output', # final output files already merged by chromosome @@ -63,8 +63,8 @@ sub default_options { step_size => 4000, # number of variants used to split the main vcf files check_transcripts => 0, # if set to 1 checks which are the new MANE Select transcripts for the last months and only calculates SpliceAI scores for these variants overlapping these transcripts transcripts_from_file => undef, - time_interval => 4, # checks which transcripts were updated/created in the last 4 months; only used if check_transcripts = 1 and we want to check the new transcripts in the core db - masked_scores => 1, # calculate masked scores + time_interval => 4, # checks which transcripts were updated/created in the last 4 months; only used if check_transcripts = 1 and we want to check the new transcripts in the core db + masked_scores => 1, # calculate masked scores registry => undef, # database where new MANE transcripts are going to be checked; only used if check_transcripts = 1 output_file_name => 'spliceai_final_scores_', @@ -83,23 +83,10 @@ sub default_options { sub resource_classes { my ($self) = @_; - - my $step_size = $self->o('step_size'); - my $time = 4; - if($step_size > 4000) { - $time = $step_size/1000; - } - return { %{$self->SUPER::resource_classes}, - '8Gb_8c_job' => { 'LSF' => '-n 8 -q production -R"select[mem>8000] rusage[mem=8000]" -M8000', - 'SLURM' => "--cpus-per-task=8 --partition=standard --time=$time:00:00 --mem=8G" - }, - '4Gb_job' => { 'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000', - 'SLURM' => "--partition=standard --time=$time:00:00 --mem=4G" - }, - 'default' => { 'LSF' => '-R"select[mem>1000] rusage[mem=1000]" -M1000', - 'SLURM' => "--partition=standard --time=1:00:00 --mem=1G" } + '8Gb_8c_job' => {'LSF' => '-n 8 -q production -R"select[mem>8000] rusage[mem=8000]" -M8000' }, + '4Gb_job' => {'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000'}, }; } From b8cceaf07e18e5fce2939458d6733517827a0f72 Mon Sep 17 00:00:00 2001 From: dglemos Date: Fri, 19 May 2023 14:31:21 +0100 Subject: [PATCH 10/13] Update variation consequence rc_name --- .../Pipeline/VariationConsequence_conf.pm | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm index c9aff9182..b583e1307 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm @@ -202,10 +202,12 @@ sub resource_classes { return { 'default' => { 'LSF' => $self->o('default_lsf_options'), 'SLURM' => $self->o('default_slurm_options') }, - 'highmem' => { 'LSF' => $self->o('highmem_lsf_options'), - 'SLURM' => $self->o('highmem_slurm_options') }, - 'medmem' => { 'LSF' => $self->o('medmem_lsf_options'), - 'SLURM' => $self->o('medmem_slurm_options') }, + 'default_long' => { 'LSF' => $self->o('default_lsf_options'), + 'SLURM' => $self->o('default_long_slurm_options') }, + 'highmem' => { 'LSF' => $self->o('highmem_lsf_options'), + 'SLURM' => $self->o('highmem_slurm_options') }, + 'medmem' => { 'LSF' => $self->o('medmem_lsf_options'), + 'SLURM' => $self->o('medmem_slurm_options') }, }; } @@ -308,7 +310,7 @@ sub pipeline_analyses { update_diff => $self->o('update_diff'), @common_params, }, - -rc_name => 'default', + -rc_name => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long', }, { -logic_name => 'transcript_effect', -module => 'Bio::EnsEMBL::Variation::Pipeline::TranscriptEffect', @@ -356,14 +358,14 @@ sub pipeline_analyses { update_diff => $self->o('update_diff'), @common_params, }, - -rc_name => 'default', + -rc_name => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long', }, { -logic_name => 'rebuild_tv_indexes', -module => 'Bio::EnsEMBL::Variation::Pipeline::RebuildIndexes', -parameters => { @common_params, }, - -rc_name => 'default', + -rc_name => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long', -wait_for => 'web_index_load', -flow_into => { 1 => ['update_variation_feature'], @@ -375,7 +377,7 @@ sub pipeline_analyses { -parameters => { @common_params, }, - -rc_name => 'default', + -rc_name => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long', -flow_into => { 1 => ['check_transcript_variation'] }, @@ -430,7 +432,7 @@ sub pipeline_analyses { }, -input_ids => [], -hive_capacity => 1, - -rc_name => 'default', + -rc_name => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long', -wait_for => [ 'set_variation_class' ], -flow_into => {}, }, From d6faa1ce9ba376e59aae66be8dab788a18cc8102 Mon Sep 17 00:00:00 2001 From: dglemos Date: Fri, 19 May 2023 14:56:24 +0100 Subject: [PATCH 11/13] Update equivalent alleles --- .../EquivalentAlleles/EquivalentAlleles_conf.pm | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/EquivalentAlleles/EquivalentAlleles_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/EquivalentAlleles/EquivalentAlleles_conf.pm index 20de991fb..e1309c042 100755 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/EquivalentAlleles/EquivalentAlleles_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/EquivalentAlleles/EquivalentAlleles_conf.pm @@ -74,10 +74,13 @@ sub default_options { # configuration for the various resource options used in the pipeline # EBI farm users should either change these here, or override them on the # command line to suit the EBI farm. - default_lsf_options => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000', medium_lsf_options => '-qproduction -R"select[mem>4000] rusage[mem=4000]" -M4000', + default_slurm_options => '--partition=standard --time=12:00:00 --mem=2G', + default_long_slurm_options => '--partition=standard --time=36:00:00 --mem=2G', + medium_slurm_options => '--partition=standard --time=12:00:00 --mem=4G', + # size of region to be checked in a single job region_size => 100000000, @@ -118,8 +121,12 @@ sub default_options { sub resource_classes { my ($self) = @_; return { - 'default' => { 'LSF' => $self->o('default_lsf_options') }, - 'medium' => { 'LSF' => $self->o('medium_lsf_options') }, + 'default' => { 'LSF' => $self->o('default_lsf_options'), + 'SLURM' => $self->o('default_slurm_options') }, + 'default_long' => { 'LSF' => $self->o('default_lsf_options'), + 'SLURM' => $self->o('default_long_slurm_options') }, + 'medium' => { 'LSF' => $self->o('medium_lsf_options'), + 'SLURM' => $self->o('medium_slurm_options') }, }; } @@ -162,7 +169,7 @@ sub pipeline_analyses { -input_ids => [], -hive_capacity => $self->o('capacity'), -max_retry_count => 0, - -rc_name => 'default', + -rc_name => 'default_long', -wait_for => [ 'init_equivalent_alleles' ], -flow_into => {}, }, From 367b0f8bc8c82428f9aba3819ff8be7ccd778194 Mon Sep 17 00:00:00 2001 From: dglemos Date: Fri, 19 May 2023 15:10:34 +0100 Subject: [PATCH 12/13] Update CAR pipeline --- .../Variation/Pipeline/GetCAR/GetCAR_conf.pm | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/GetCAR/GetCAR_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/GetCAR/GetCAR_conf.pm index c7eec68cd..93b2d7132 100755 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/GetCAR/GetCAR_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/GetCAR/GetCAR_conf.pm @@ -87,10 +87,14 @@ sub resource_classes { my ($self) = @_; return { %{$self->SUPER::resource_classes}, # inherit 'default' from the parent class - 'test_mem' => { 'LSF' => '-q production -R"select[mem>100] rusage[mem=100]" -M100'}, - 'default_mem' => { 'LSF' => '-q production -R"select[mem>1000] rusage[mem=1000]" -M1000'}, - 'medium_mem' => { 'LSF' => '-q production -R"select[mem>2000] rusage[mem=2000]" -M2000'}, - 'high_mem' => { 'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000'}, + 'test_mem' => { 'LSF' => '-q production -R"select[mem>100] rusage[mem=100]" -M100', + 'SLURM' => '--partition=standard --time=12:00:00 --mem=100MB' }, + 'default_mem' => { 'LSF' => '-q production -R"select[mem>1000] rusage[mem=1000]" -M1000', + 'SLURM' => '--partition=standard --time=12:00:00 --mem=1G' }, + 'medium_mem' => { 'LSF' => '-q production -R"select[mem>2000] rusage[mem=2000]" -M2000', + 'SLURM' => '--partition=standard --time=12:00:00 --mem=2G'}, + 'high_mem' => { 'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000', + 'SLURM' => '--partition=standard --time=12:00:00 --mem=4G' }, }; } @@ -127,6 +131,7 @@ sub pipeline_analyses { { -logic_name => 'finish_get_car', -module => 'Bio::EnsEMBL::Variation::Pipeline::GetCAR::FinishGetCAR', + -rc_name => 'default_mem', -max_retry_count => 0, }, ); From e7e5556f3f1066008a6fe52972b43cc65940968c Mon Sep 17 00:00:00 2001 From: dglemos Date: Fri, 19 May 2023 15:20:04 +0100 Subject: [PATCH 13/13] Update ancestral allele pipeline --- .../Pipeline/AncestralAlleles/AncestralAlleles_conf.pm | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/AncestralAlleles/AncestralAlleles_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/AncestralAlleles/AncestralAlleles_conf.pm index 4a1e8163b..4d84cd447 100644 --- a/modules/Bio/EnsEMBL/Variation/Pipeline/AncestralAlleles/AncestralAlleles_conf.pm +++ b/modules/Bio/EnsEMBL/Variation/Pipeline/AncestralAlleles/AncestralAlleles_conf.pm @@ -82,6 +82,9 @@ sub default_options { default_lsf_options => '-q production -R"select[mem>2000] rusage[mem=2000]" -M2000', medmem_lsf_options => '-q production -R"select[mem>5000] rusage[mem=5000]" -M5000', + default_slurm_options => '--partition=standard --time=12:00:00 --mem=2G', + medmem_slurm_options => '--partition=standard --time=12:00:00 --mem=5G', + hive_db_host => 'mysql-ens-var-prod-1', hive_db_port => 4449, hive_db_user => 'ensadmin', @@ -101,8 +104,10 @@ sub default_options { sub resource_classes { my ($self) = @_; return { - 'default' => { 'LSF' => $self->o('default_lsf_options') }, - 'medmem' => { 'LSF' => $self->o('medmem_lsf_options') }, + 'default' => { 'LSF' => $self->o('default_lsf_options'), + 'SLURM' => $self->o('default_slurm_options') }, + 'medmem' => { 'LSF' => $self->o('medmem_lsf_options'), + 'SLURM' => $self->o('medmem_slurm_options') }, }; } @@ -157,4 +162,3 @@ sub pipeline_analyses { } 1; -