From c64e2b7e043ba3bf4989e701a057d0b3b5fb0209 Mon Sep 17 00:00:00 2001
From: Leo Gordon <lg4@ebi.ac.uk>
Date: Fri, 16 Nov 2012 12:11:01 +0000
Subject: [PATCH] make Valley into Configurable and move SubmitWorkersMax into
Valley's context, because it is more "global" than a Meadow
---
hive_config.json | 9 ++++++---
modules/Bio/EnsEMBL/Hive/Queen.pm | 2 +-
modules/Bio/EnsEMBL/Hive/Valley.pm | 5 +++++
scripts/beekeeper.pl | 2 +-
4 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/hive_config.json b/hive_config.json
index 0d4cbfa02..0c40fabd5 100644
--- a/hive_config.json
+++ b/hive_config.json
@@ -1,16 +1,19 @@
{
+ "Valley" : {
+ "SubmitWorkersMax" : 50, # if set, provides a cut-off on the total number of workers (across all Meadows) submitted each iteration
+ },
"Meadow" : { # configure Meadow-specific options here:
"LSF" : {
"SubmissionOptions" : "", # general options that different Meadows can plug into the submission command
"TotalRunningWorkersMax" : 1000, # how many workers can be run on this Meadow in total
"farm2" : {
- "SubmitWorkersMax" : 50, # if set, provides a cut-off on the number of workers being submitted each iteration
+ "SubmissionOptions" : "", # submission options specific for LSF/farm2
},
"pcs4" : {
- "SubmitWorkersMax" : 30,
+ "SubmissionOptions" : "", # submission options specific for LSF/pcs4
},
"EBI" : {
- "SubmitWorkersMax" : 60,
+ "SubmissionOptions" : "", # submission options specific for LSF/EBI
},
},
"LOCAL" : {
diff --git a/modules/Bio/EnsEMBL/Hive/Queen.pm b/modules/Bio/EnsEMBL/Hive/Queen.pm
index 69dd1ab1e..53718f303 100644
--- a/modules/Bio/EnsEMBL/Hive/Queen.pm
+++ b/modules/Bio/EnsEMBL/Hive/Queen.pm
@@ -814,10 +814,10 @@ sub schedule_workers {
sub schedule_workers_resync_if_necessary {
my ($self, $valley, $analysis) = @_;
+ my $submit_limit = $valley->config_get('SubmitWorkersMax');
my $meadow = $valley->get_current_meadow();
my $pending_by_rc_name = $meadow->can('count_pending_workers_by_rc_name') ? $meadow->count_pending_workers_by_rc_name() : {};
- my $submit_limit = $meadow->config_get('SubmitWorkersMax');
my $meadow_limit = ($meadow->can('count_running_workers') and defined($meadow->config_get('TotalRunningWorkersMax'))) ? $meadow->config_get('TotalRunningWorkersMax') - $meadow->count_running_workers : undef;
my $available_submit_limit = ($submit_limit and $meadow_limit)
diff --git a/modules/Bio/EnsEMBL/Hive/Valley.pm b/modules/Bio/EnsEMBL/Hive/Valley.pm
index 460c37d7c..7c74fab8a 100644
--- a/modules/Bio/EnsEMBL/Hive/Valley.pm
+++ b/modules/Bio/EnsEMBL/Hive/Valley.pm
@@ -26,6 +26,8 @@ use warnings;
use Sys::Hostname;
use Bio::EnsEMBL::Hive::Utils ('find_submodules');
+use base ('Bio::EnsEMBL::Hive::Configurable');
+
sub meadow_class_path {
@@ -38,6 +40,9 @@ sub new {
my $self = bless {}, $class;
+ $self->config( $config );
+ $self->context( [ 'Valley' ] );
+
my $amch = $self->available_meadow_hash( {} );
# make sure modules are loaded and available ones are checked prior to setting the current one
diff --git a/scripts/beekeeper.pl b/scripts/beekeeper.pl
index d17ad9c39..701e285d8 100755
--- a/scripts/beekeeper.pl
+++ b/scripts/beekeeper.pl
@@ -156,12 +156,12 @@ sub main {
$meadow_type = 'LOCAL' if($local);
my $valley = Bio::EnsEMBL::Hive::Valley->new( $config, $meadow_type, $pipeline_name );
+ $valley->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
my $current_meadow = $valley->get_current_meadow();
warn "Current meadow: ".$current_meadow->signature."\n\n";
$current_meadow->config_set('TotalRunningWorkersMax', $total_running_workers_max) if(defined $total_running_workers_max);
- $current_meadow->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
$current_meadow->config_set('SubmissionOptions', $submission_options) if(defined $submission_options);
if($reset_job_id) { $queen->reset_job_by_dbID_and_sync($reset_job_id); }
--
GitLab