Commit c64e2b7e authored by Leo Gordon's avatar Leo Gordon
Browse files

make Valley into Configurable and move SubmitWorkersMax into Valley's context,...

make Valley into Configurable and move SubmitWorkersMax into Valley's context, because it is more "global" than a Meadow
parent 93df162a
{ {
"Valley" : {
"SubmitWorkersMax" : 50, # if set, provides a cut-off on the total number of workers (across all Meadows) submitted each iteration
},
"Meadow" : { # configure Meadow-specific options here: "Meadow" : { # configure Meadow-specific options here:
"LSF" : { "LSF" : {
"SubmissionOptions" : "", # general options that different Meadows can plug into the submission command "SubmissionOptions" : "", # general options that different Meadows can plug into the submission command
"TotalRunningWorkersMax" : 1000, # how many workers can be run on this Meadow in total "TotalRunningWorkersMax" : 1000, # how many workers can be run on this Meadow in total
"farm2" : { "farm2" : {
"SubmitWorkersMax" : 50, # if set, provides a cut-off on the number of workers being submitted each iteration "SubmissionOptions" : "", # submission options specific for LSF/farm2
}, },
"pcs4" : { "pcs4" : {
"SubmitWorkersMax" : 30, "SubmissionOptions" : "", # submission options specific for LSF/pcs4
}, },
"EBI" : { "EBI" : {
"SubmitWorkersMax" : 60, "SubmissionOptions" : "", # submission options specific for LSF/EBI
}, },
}, },
"LOCAL" : { "LOCAL" : {
......
...@@ -814,10 +814,10 @@ sub schedule_workers { ...@@ -814,10 +814,10 @@ sub schedule_workers {
sub schedule_workers_resync_if_necessary { sub schedule_workers_resync_if_necessary {
my ($self, $valley, $analysis) = @_; my ($self, $valley, $analysis) = @_;
my $submit_limit = $valley->config_get('SubmitWorkersMax');
my $meadow = $valley->get_current_meadow(); my $meadow = $valley->get_current_meadow();
my $pending_by_rc_name = $meadow->can('count_pending_workers_by_rc_name') ? $meadow->count_pending_workers_by_rc_name() : {}; my $pending_by_rc_name = $meadow->can('count_pending_workers_by_rc_name') ? $meadow->count_pending_workers_by_rc_name() : {};
my $submit_limit = $meadow->config_get('SubmitWorkersMax');
my $meadow_limit = ($meadow->can('count_running_workers') and defined($meadow->config_get('TotalRunningWorkersMax'))) ? $meadow->config_get('TotalRunningWorkersMax') - $meadow->count_running_workers : undef; my $meadow_limit = ($meadow->can('count_running_workers') and defined($meadow->config_get('TotalRunningWorkersMax'))) ? $meadow->config_get('TotalRunningWorkersMax') - $meadow->count_running_workers : undef;
my $available_submit_limit = ($submit_limit and $meadow_limit) my $available_submit_limit = ($submit_limit and $meadow_limit)
......
...@@ -26,6 +26,8 @@ use warnings; ...@@ -26,6 +26,8 @@ use warnings;
use Sys::Hostname; use Sys::Hostname;
use Bio::EnsEMBL::Hive::Utils ('find_submodules'); use Bio::EnsEMBL::Hive::Utils ('find_submodules');
use base ('Bio::EnsEMBL::Hive::Configurable');
sub meadow_class_path { sub meadow_class_path {
...@@ -38,6 +40,9 @@ sub new { ...@@ -38,6 +40,9 @@ sub new {
my $self = bless {}, $class; my $self = bless {}, $class;
$self->config( $config );
$self->context( [ 'Valley' ] );
my $amch = $self->available_meadow_hash( {} ); my $amch = $self->available_meadow_hash( {} );
# make sure modules are loaded and available ones are checked prior to setting the current one # make sure modules are loaded and available ones are checked prior to setting the current one
......
...@@ -156,12 +156,12 @@ sub main { ...@@ -156,12 +156,12 @@ sub main {
$meadow_type = 'LOCAL' if($local); $meadow_type = 'LOCAL' if($local);
my $valley = Bio::EnsEMBL::Hive::Valley->new( $config, $meadow_type, $pipeline_name ); my $valley = Bio::EnsEMBL::Hive::Valley->new( $config, $meadow_type, $pipeline_name );
$valley->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
my $current_meadow = $valley->get_current_meadow(); my $current_meadow = $valley->get_current_meadow();
warn "Current meadow: ".$current_meadow->signature."\n\n"; warn "Current meadow: ".$current_meadow->signature."\n\n";
$current_meadow->config_set('TotalRunningWorkersMax', $total_running_workers_max) if(defined $total_running_workers_max); $current_meadow->config_set('TotalRunningWorkersMax', $total_running_workers_max) if(defined $total_running_workers_max);
$current_meadow->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
$current_meadow->config_set('SubmissionOptions', $submission_options) if(defined $submission_options); $current_meadow->config_set('SubmissionOptions', $submission_options) if(defined $submission_options);
if($reset_job_id) { $queen->reset_job_by_dbID_and_sync($reset_job_id); } if($reset_job_id) { $queen->reset_job_by_dbID_and_sync($reset_job_id); }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment