Commit c64e2b7e authored by Leo Gordon's avatar Leo Gordon
Browse files

make Valley into Configurable and move SubmitWorkersMax into Valley's context,...

make Valley into Configurable and move SubmitWorkersMax into Valley's context, because it is more "global" than a Meadow
parent 93df162a
{
"Valley" : {
"SubmitWorkersMax" : 50, # if set, provides a cut-off on the total number of workers (across all Meadows) submitted each iteration
},
"Meadow" : { # configure Meadow-specific options here:
"LSF" : {
"SubmissionOptions" : "", # general options that different Meadows can plug into the submission command
"TotalRunningWorkersMax" : 1000, # how many workers can be run on this Meadow in total
"farm2" : {
"SubmitWorkersMax" : 50, # if set, provides a cut-off on the number of workers being submitted each iteration
"SubmissionOptions" : "", # submission options specific for LSF/farm2
},
"pcs4" : {
"SubmitWorkersMax" : 30,
"SubmissionOptions" : "", # submission options specific for LSF/pcs4
},
"EBI" : {
"SubmitWorkersMax" : 60,
"SubmissionOptions" : "", # submission options specific for LSF/EBI
},
},
"LOCAL" : {
......
......@@ -814,10 +814,10 @@ sub schedule_workers {
sub schedule_workers_resync_if_necessary {
my ($self, $valley, $analysis) = @_;
my $submit_limit = $valley->config_get('SubmitWorkersMax');
my $meadow = $valley->get_current_meadow();
my $pending_by_rc_name = $meadow->can('count_pending_workers_by_rc_name') ? $meadow->count_pending_workers_by_rc_name() : {};
my $submit_limit = $meadow->config_get('SubmitWorkersMax');
my $meadow_limit = ($meadow->can('count_running_workers') and defined($meadow->config_get('TotalRunningWorkersMax'))) ? $meadow->config_get('TotalRunningWorkersMax') - $meadow->count_running_workers : undef;
my $available_submit_limit = ($submit_limit and $meadow_limit)
......
......@@ -26,6 +26,8 @@ use warnings;
use Sys::Hostname;
use Bio::EnsEMBL::Hive::Utils ('find_submodules');
use base ('Bio::EnsEMBL::Hive::Configurable');
sub meadow_class_path {
......@@ -38,6 +40,9 @@ sub new {
my $self = bless {}, $class;
$self->config( $config );
$self->context( [ 'Valley' ] );
my $amch = $self->available_meadow_hash( {} );
# make sure modules are loaded and available ones are checked prior to setting the current one
......
......@@ -156,12 +156,12 @@ sub main {
$meadow_type = 'LOCAL' if($local);
my $valley = Bio::EnsEMBL::Hive::Valley->new( $config, $meadow_type, $pipeline_name );
$valley->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
my $current_meadow = $valley->get_current_meadow();
warn "Current meadow: ".$current_meadow->signature."\n\n";
$current_meadow->config_set('TotalRunningWorkersMax', $total_running_workers_max) if(defined $total_running_workers_max);
$current_meadow->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
$current_meadow->config_set('SubmissionOptions', $submission_options) if(defined $submission_options);
if($reset_job_id) { $queen->reset_job_by_dbID_and_sync($reset_job_id); }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment