Commit 6c84a67c authored by Leo Gordon's avatar Leo Gordon
Browse files

a new example pipeline designed to fail because of MEMLIMIT in some of the cases

parent 9b557013
=pod
=head1 NAME
Bio::EnsEMBL::Hive::PipeConfig::MemlimitTest_conf
=head1 SYNOPSIS
init_pipeline.pl Bio::EnsEMBL::Hive::PipeConfig::MemlimitTest_conf -password <your_password>
=head1 DESCRIPTION
This is another example pipeline built around FailureTest.pm RunnableDB. It consists of two analyses:
Analysis_1: JobFactory.pm is used to create an array of jobs -
these jobs are sent down the branch #2 into the second analysis
Analysis_2: FailureTest.pm in "memory grab" mode may overrun the current resource's memory limit and be killed by the LSF
=head1 CONTACT
Please contact ehive-users@ebi.ac.uk mailing list with questions/suggestions.
=cut
package Bio::EnsEMBL::Hive::PipeConfig::MemlimitTest_conf;
use strict;
use warnings;
use base ('Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf'); # All Hive databases configuration files should inherit from HiveGeneric, directly or indirectly
sub default_options {
my ($self) = @_;
return {
%{ $self->SUPER::default_options() }, # inherit other stuff from the base class
'pipeline_name' => 'memlimit_test', # name used by the beekeeper to prefix job names on the farm
};
}
sub resource_classes {
my ($self) = @_;
return {
%{$self->SUPER::resource_classes}, # inherit 'default' from the parent class
'200Mb_job' => {'LSF' => '-C0 -M200 -R"select[mem>200] rusage[mem=200]"' },
'400Mb_job' => {'LSF' => '-C0 -M400 -R"select[mem>400] rusage[mem=400]"' },
'1Gb_job' => {'LSF' => '-C0 -M1000 -R"select[mem>1000] rusage[mem=1000]"' },
};
}
sub pipeline_analyses {
my ($self) = @_;
return [
{ -logic_name => 'generate_jobs',
-module => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
-meadow_type => 'LOCAL',
-parameters => {
'column_names' => [ 'grab_mln' ],
},
-input_ids => [
{ 'inputlist' => [ 0.6 , 0.8 , 1.0 , 1.2 , 1.4 , 1.6 , 1.8 , 2.0, 2.5, 3.0, 5.0, 7.0, 10 ], },
],
-flow_into => {
2 => [ 'failure_test' ],
},
},
{ -logic_name => 'failure_test',
-module => 'Bio::EnsEMBL::Hive::RunnableDB::FailureTest',
-parameters => {
'time_RUN' => 30,
},
-flow_into => {
-1 => [ 'failure_test_himem' ],
}
},
{ -logic_name => 'failure_test_himem',
-module => 'Bio::EnsEMBL::Hive::RunnableDB::FailureTest',
-parameters => {
'time_RUN' => 30,
},
-rc_name => '200Mb_job',
},
];
}
1;
......@@ -67,12 +67,14 @@ sub param_defaults {
return {
'value' => 1, # normally you generate a batch of jobs with different values of param('value')
'divisor' => 2, # but the same param('divisor') and see how every param('divisor')'s job will crash
'state' => 'RUN', # the state in which the process may commit apoptosis
'state' => 'RUN', # the state in which the process may commit apoptosis ('FETCH_INPUT', 'RUN' or 'WRITE_OUTPUT')
'lethal_after' => 0, # If value is above this (nonzero) threshold, job's death becomes lethal to the worker.
'time_FETCH_INPUT' => 0, # how much time fetch_input() will spend in sleeping state
'time_RUN' => 1, # how much time run() will spend in sleeping state
'time_WRITE_OUTPUT' => 0, # how much time write_output() will spend in sleeping state
'grab_mln' => 0, # how many millions of numeric elements to allocate
};
}
......@@ -113,6 +115,11 @@ sub fetch_input {
sub run {
my $self = shift @_;
my $mem_ref;
if(my $grab_mln = $self->param('grab_mln')) {
$mem_ref = $self->grab_memory( $grab_mln );
}
$self->dangerous_math('RUN');
}
......@@ -155,7 +162,8 @@ sub dangerous_math {
my ($self, $current_state) = @_;
# First, sleep as required:
sleep($self->param('time_'.$current_state));
my $seconds_to_sleep = $self->param('time_'.$current_state);
sleep( $seconds_to_sleep );
my $state = $self->param('state');
return if($current_state ne $state);
......@@ -175,5 +183,21 @@ sub dangerous_math {
}
}
sub grab_memory {
my ($self, $grab_mln) = @_;
my $elements = $grab_mln*1_000_000;
my $estimated_megs = $grab_mln*69 + 23; # empirically found by running on farm3, may differ elsewhere
$|=1;
$self->warning("Allocating $elements elements, which should map to approximately $estimated_megs megabytes");
my @mem = (1..$elements);
return \@mem;
}
1;
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment