a new example pipeline designed to fail because of MEMLIMIT in some of the cases

modules/Bio/EnsEMBL/Hive/PipeConfig/MemlimitTest_conf.pm

+
+=pod
+
+=head1 NAME
+
+  Bio::EnsEMBL::Hive::PipeConfig::MemlimitTest_conf
+
+=head1 SYNOPSIS
+
+    init_pipeline.pl Bio::EnsEMBL::Hive::PipeConfig::MemlimitTest_conf -password <your_password>
+
+=head1 DESCRIPTION
+
+    This is another example pipeline built around FailureTest.pm RunnableDB. It consists of two analyses:
+
+    Analysis_1: JobFactory.pm is used to create an array of jobs -
+
+        these jobs are sent down the branch #2 into the second analysis
+
+    Analysis_2: FailureTest.pm in "memory grab" mode may overrun the current resource's memory limit and be killed by the LSF
+
+=head1 CONTACT
+
+  Please contact ehive-users@ebi.ac.uk mailing list with questions/suggestions.
+
+=cut
+
+package Bio::EnsEMBL::Hive::PipeConfig::MemlimitTest_conf;
+
+use strict;
+use warnings;
+
+use base ('Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf');  # All Hive databases configuration files should inherit from HiveGeneric, directly or indirectly
+
+
+sub default_options {
+    my ($self) = @_;
+    return {
+        %{ $self->SUPER::default_options() },               # inherit other stuff from the base class
+
+        'pipeline_name' => 'memlimit_test',                 # name used by the beekeeper to prefix job names on the farm
+    };
+}
+
+
+sub resource_classes {
+    my ($self) = @_;
+    return {
+        %{$self->SUPER::resource_classes},  # inherit 'default' from the parent class
+
+         '200Mb_job'    => {'LSF' => '-C0 -M200   -R"select[mem>200]   rusage[mem=200]"' },
+         '400Mb_job'    => {'LSF' => '-C0 -M400   -R"select[mem>400]   rusage[mem=400]"' },
+         '1Gb_job'      => {'LSF' => '-C0 -M1000  -R"select[mem>1000]  rusage[mem=1000]"' },
+    };
+}
+
+
+sub pipeline_analyses {
+    my ($self) = @_;
+    return [
+        {   -logic_name => 'generate_jobs',
+            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
+            -meadow_type => 'LOCAL',
+            -parameters => {
+                'column_names' => [ 'grab_mln' ],
+            },
+            -input_ids => [
+                { 'inputlist' => [ 0.6 , 0.8 , 1.0 , 1.2 , 1.4 , 1.6 , 1.8 , 2.0, 2.5, 3.0, 5.0, 7.0, 10 ], },
+            ],
+
+            -flow_into => {
+                2 => [ 'failure_test' ],
+            },
+        },
+
+        {   -logic_name    => 'failure_test',
+            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::FailureTest',
+            -parameters => {
+                'time_RUN'      => 30,
+            },
+            -flow_into => {
+                -1 => [ 'failure_test_himem' ],
+            }
+        },
+
+        {   -logic_name    => 'failure_test_himem',
+            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::FailureTest',
+            -parameters => {
+                'time_RUN'      => 30,
+            },
+            -rc_name => '200Mb_job',
+        },
+    ];
+}
+
+1;
+

modules/Bio/EnsEMBL/Hive/RunnableDB/FailureTest.pm

@@ -67,12 +67,14 @@ sub param_defaults {
     return {
         'value'         => 1,       # normally you generate a batch of jobs with different values of param('value')
         'divisor'       => 2,       # but the same param('divisor') and see how every param('divisor')'s job will crash
-        'state'         => 'RUN',   # the state in which the process may commit apoptosis
+        'state'         => 'RUN',   # the state in which the process may commit apoptosis ('FETCH_INPUT', 'RUN' or 'WRITE_OUTPUT')
         'lethal_after'  => 0,       # If value is above this (nonzero) threshold, job's death becomes lethal to the worker.
 
         'time_FETCH_INPUT'  => 0,   # how much time fetch_input()  will spend in sleeping state
         'time_RUN'          => 1,   # how much time run()          will spend in sleeping state
         'time_WRITE_OUTPUT' => 0,   # how much time write_output() will spend in sleeping state
+
+        'grab_mln'          => 0,   # how many millions of numeric elements to allocate
     };
 }
 
@@ -113,6 +115,11 @@ sub fetch_input {
 sub run {
     my $self = shift @_;
 
+    my $mem_ref;
+    if(my $grab_mln = $self->param('grab_mln')) {
+        $mem_ref = $self->grab_memory( $grab_mln );
+    }
+
     $self->dangerous_math('RUN');
 }
 
@@ -155,7 +162,8 @@ sub dangerous_math {
     my ($self, $current_state) = @_;
 
         # First, sleep as required:
-    sleep($self->param('time_'.$current_state));
+    my $seconds_to_sleep = $self->param('time_'.$current_state);
+    sleep( $seconds_to_sleep );
 
     my $state   = $self->param('state');
     return if($current_state ne $state);
@@ -175,5 +183,21 @@ sub dangerous_math {
     }
 }
 
+
+sub grab_memory {
+    my ($self, $grab_mln) = @_;
+
+    my $elements        = $grab_mln*1_000_000;
+    my $estimated_megs  = $grab_mln*69 + 23;    # empirically found by running on farm3, may differ elsewhere
+
+    $|=1;
+
+    $self->warning("Allocating $elements elements, which should map to approximately $estimated_megs megabytes");
+
+    my @mem = (1..$elements);
+
+    return \@mem;
+}
+
 1;