an offline test to create deadlocks on semaphore_counts during transa…

…ctions

t/10.pipeconfig/TestPipeConfig/SemaCounterOverload_conf.pm

@@ -0,0 +1,119 @@
+=pod
+
+=head1 NAME
+
+    TestPipeConfig::SemaCounterOverload_conf
+
+=head1 DESCRIPTION
+
+    This is a PipeConfig for a special "stress-inducing" pipeline that intentionally creates a very high load
+    on the semaphore counters to assess the behaviour and efficiency of or our deadlock-avoiding approaches.
+
+    It has to be "pessimized" for a specific farm/cluster (EBI RH7), so does not take part in an automatic test and is run manually.
+
+    In order to create deadlocks comment out the prelock_ calls before inserting a Job in AnalysisJobAdaptor,
+    then run two separate workers to saturate the 'fan_C' analysis (takes under a minunte),
+    and finally submit 300 Workers of analysis 'fan_C' in one go by running beepeeker.pl with "-submit_workers_max 300 -run" .
+
+    You get the best results (more collisions) when the farm is not busy and all the 300 workers run in parallel.
+
+=head1 LICENSE
+
+    Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+    Copyright [2016] EMBL-European Bioinformatics Institute
+
+    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software distributed under the License
+    is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and limitations under the License.
+
+=head1 CONTACT
+
+    Please subscribe to the Hive mailing list:  http://listserver.ebi.ac.uk/mailman/listinfo/ehive-users  to discuss Hive-related questions or to be notified of our updates
+
+=cut
+
+
+package TestPipeConfig::SemaCounterOverload_conf;
+
+use strict;
+use warnings;
+
+use base ('Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf');  # All Hive databases configuration files should inherit from HiveGeneric, directly or indirectly
+use Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf;           # Allow this particular config to use conditional dataflow and INPUT_PLUS
+
+
+sub default_options {
+    my ($self) = @_;
+    return {
+        %{ $self->SUPER::default_options() },
+
+        'time'              => '40+30*rand(1)',
+        'num_semaphores'    => 1,
+        'num_per_sem'       => 3000,
+        'capacity'          => 300,
+    };
+}
+
+sub pipeline_analyses {
+    my ($self) = @_;
+    return [
+        {   -logic_name => 'factory_A',
+            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
+            -meadow_type => 'LOCAL',
+            -parameters => {
+                'inputlist'    => '#expr([1..#num_semaphores#])expr#',
+                'column_names' => [ 'sem_index' ],
+            },
+            -input_ids => [
+                { 'num_semaphores' => $self->o('num_semaphores') },
+            ],
+            -analysis_capacity => 1,
+            -flow_into => {
+                '2->A' => 'factory_B',
+                'A->1' => 'funnel_E',
+            },
+        },
+
+        {   -logic_name     => 'factory_B',
+            -module         => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
+            -parameters     => {
+                'inputlist'     => '#expr([1..#num_per_sem#])expr#',
+                'column_names'  => [ 'sem_subindex' ],
+                'num_per_sem'   => $self->o('num_per_sem'),
+            },
+            -flow_into => {
+                '2' => { 'fan_C' => INPUT_PLUS() },
+            }
+        },
+
+        {   -logic_name    => 'fan_C',
+            -module        => 'TestRunnable::TransactDummy',
+            -hive_capacity  => $self->o('capacity'),
+            -parameters    => {
+                'take_time'         => $self->o('time'),
+            },
+            -flow_into => {
+                '1' => 'fan_D',
+            }
+        },
+
+        {   -logic_name    => 'fan_D',
+            -module        => 'TestRunnable::TransactDummy',
+            -hive_capacity  => $self->o('capacity'),
+            -parameters    => {
+                'take_time'         => $self->o('time'),
+            },
+        },
+
+        {   -logic_name    => 'funnel_E',
+            -module        => 'TestRunnable::TransactDummy',
+        },
+    ];
+}
+
+1;

t/10.pipeconfig/TestRunnable/TransactDummy.pm

@@ -0,0 +1,67 @@
+=pod 
+
+=head1 NAME
+
+    TestRunnable::TransactDummy
+
+=head1 DESCRIPTION
+
+    A job of 'TestRunnable::TransactDummy' analysis does not do any work,
+    but it sleeps some macroscopic amount of time within a relatively long transaction
+    that also performs critical semaphore_count updates.
+
+    We use this TestRunnable within OverloadTest_conf pipeline to study the effects
+    of transactional deadlocks and develop a better solution against them.
+
+=head1 LICENSE
+
+    Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+    Copyright [2016] EMBL-European Bioinformatics Institute
+
+    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software distributed under the License
+    is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and limitations under the License.
+
+=head1 CONTACT
+
+    Please subscribe to the Hive mailing list:  http://listserver.ebi.ac.uk/mailman/listinfo/ehive-users  to discuss Hive-related questions or to be notified of our updates
+
+=cut
+
+
+package TestRunnable::TransactDummy;
+
+use strict;
+use warnings;
+use Time::HiRes ('usleep');
+
+use base ('Bio::EnsEMBL::Hive::Process');
+
+sub param_defaults {
+
+    return {
+        'take_time' => 0,   # how much time run() method will spend in sleeping state
+    };
+}
+
+
+sub run {
+    my $self = shift @_;
+
+    my $take_time = eval $self->param('take_time');
+    if($take_time) {
+        $self->dbc->run_in_transaction( sub {
+            print "Sleeping for '$take_time' seconds...\n";
+            usleep( $take_time*1000000 );
+            print "Done.\n";
+            $self->dataflow_output_id(undef, 1);
+        } );
+    }
+}
+
+1;

t/10.pipeconfig/sema_counter_overload.mt

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# Copyright [2016] EMBL-European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a farm-specific manual test driver for our stress-test pipeline.
+# It is supposed to submit many workers that will cause deadlocks in the database.
+
+
+if [ -z "$1" ]; then
+    echo "Please provide pipeline_url as the only command line parameter"
+    exit 1;
+fi
+
+pipeline_url="$1"
+
+init_pipeline.pl TestPipeConfig/SemaCounterOverload_conf.pm -pipeline_url $pipeline_url -hive_force_init 1
+
+runWorker.pl -url $pipeline_url
+
+runWorker.pl -url $pipeline_url
+
+beekeeper.pl -url $pipeline_url -submit_workers_max 300 -run
+
+echo "The workers have been submitted, please check the database $pipeline_url manually and don't forget to drop it afterwards"
+