Commit 346613a2 authored by Leo Gordon's avatar Leo Gordon
Browse files

bugfix+feature: -local_cpus renamed into -total_workers_max and so made...

bugfix+feature: -local_cpus renamed into -total_workers_max and so made available for any meadow (not just LOCAL). Plus some renames
parent 337fee35
...@@ -95,14 +95,19 @@ sub kill_worker { ...@@ -95,14 +95,19 @@ sub kill_worker {
# --------------[(combinable) means of adjusting the number of submitted workers]---------------------- # --------------[(combinable) means of adjusting the number of submitted workers]----------------------
sub total_running_workers_limit { # if set and ->can('count_running_workers'), sub total_running_workers_default_max { # no default by default :)
return undef;
}
sub total_running_workers_max { # if set and ->can('count_running_workers'),
# provides a cut-off on the number of workers being submitted # provides a cut-off on the number of workers being submitted
my $self = shift @_; my $self = shift @_;
if(scalar(@_)) { # new value is being set (which can be undef) if(scalar(@_)) { # new value is being set (which can be undef)
$self->{'_total_running_workers_limit'} = shift @_; $self->{'_total_running_workers_max'} = shift @_;
} }
return $self->{'_total_running_workers_limit'}; return $self->{'_total_running_workers_max'} || $self->total_running_workers_default_max();
} }
sub pending_adjust { # if set and ->can('count_pending_workers_by_rc_id'), sub pending_adjust { # if set and ->can('count_pending_workers_by_rc_id'),
...@@ -115,13 +120,13 @@ sub pending_adjust { # if set and ->can('count_pending_workers_by_rc_id'), ...@@ -115,13 +120,13 @@ sub pending_adjust { # if set and ->can('count_pending_workers_by_rc_id'),
return $self->{'_pending_adjust'}; return $self->{'_pending_adjust'};
} }
sub submitted_workers_limit { # if set, provides a cut-off on the number of workers being submitted sub submit_workers_max { # if set, provides a cut-off on the number of workers being submitted
my $self = shift @_; my $self = shift @_;
if(scalar(@_)) { # new value is being set (which can be undef) if(scalar(@_)) { # new value is being set (which can be undef)
$self->{'_submitted_workers_limit'} = shift @_; $self->{'_submit_workers_max'} = shift @_;
} }
return $self->{'_submitted_workers_limit'}; return $self->{'_submit_workers_max'};
} }
1; 1;
...@@ -13,6 +13,13 @@ sub available { # always invoked as a class method ...@@ -13,6 +13,13 @@ sub available { # always invoked as a class method
return 1; # for the moment assume LOCAL meadow is always available return 1; # for the moment assume LOCAL meadow is always available
} }
sub total_running_workers_default_max {
return 2;
}
sub get_current_worker_process_id { sub get_current_worker_process_id {
my ($self) = @_; my ($self) = @_;
......
...@@ -744,8 +744,8 @@ sub schedule_workers_resync_if_necessary { ...@@ -744,8 +744,8 @@ sub schedule_workers_resync_if_necessary {
my ($self, $meadow, $analysis) = @_; my ($self, $meadow, $analysis) = @_;
my $pending_by_rc_id = ($meadow->can('count_pending_workers_by_rc_id') and $meadow->pending_adjust()) ? $meadow->count_pending_workers_by_rc_id() : {}; my $pending_by_rc_id = ($meadow->can('count_pending_workers_by_rc_id') and $meadow->pending_adjust()) ? $meadow->count_pending_workers_by_rc_id() : {};
my $submit_limit = $meadow->submitted_workers_limit(); my $submit_limit = $meadow->submit_workers_max();
my $meadow_limit = ($meadow->can('count_running_workers') and defined($meadow->total_running_workers_limit)) ? $meadow->total_running_workers_limit - $meadow->count_running_workers : undef; my $meadow_limit = ($meadow->can('count_running_workers') and defined($meadow->total_running_workers_max)) ? $meadow->total_running_workers_max - $meadow->count_running_workers : undef;
my $available_submit_limit = ($submit_limit and $meadow_limit) my $available_submit_limit = ($submit_limit and $meadow_limit)
? (($submit_limit<$meadow_limit) ? $submit_limit : $meadow_limit) ? (($submit_limit<$meadow_limit) ? $submit_limit : $meadow_limit)
......
...@@ -46,8 +46,8 @@ sub main { ...@@ -46,8 +46,8 @@ sub main {
my $local = 0; my $local = 0;
my $show_failed_jobs = 0; my $show_failed_jobs = 0;
my $no_pend_adjust = 0; my $no_pend_adjust = 0;
my $worker_limit = 50; my $submit_workers_max = 50;
my $local_cpus = 2; my $total_workers_max = undef;
my $meadow_name = ''; my $meadow_name = '';
my $meadow_options = ''; my $meadow_options = '';
my $run = 0; my $run = 0;
...@@ -93,8 +93,8 @@ sub main { ...@@ -93,8 +93,8 @@ sub main {
# meadow control # meadow control
'local!' => \$local, 'local!' => \$local,
'local_cpus=i' => \$local_cpus, 'total_workers_max|local_cpus=i' => \$total_workers_max,
'wlimit=i' => \$worker_limit, 'submit_workers_max|wlimit=i' => \$submit_workers_max,
'no_pend' => \$no_pend_adjust, 'no_pend' => \$no_pend_adjust,
'meadow_name=s' => \$meadow_name, 'meadow_name=s' => \$meadow_name,
'meadow_options=s' => \$meadow_options, 'meadow_options=s' => \$meadow_options,
...@@ -205,21 +205,20 @@ sub main { ...@@ -205,21 +205,20 @@ sub main {
$meadow_name = 'Bio::EnsEMBL::Hive::Meadow::'.uc($meadow_name) unless($meadow_name=~/::/); $meadow_name = 'Bio::EnsEMBL::Hive::Meadow::'.uc($meadow_name) unless($meadow_name=~/::/);
my $meadow_object; unless($available_meadow_classes{$meadow_name}) {
if($available_meadow_classes{$meadow_name}) {
warn "Current meadow: '$meadow_name'\n";
$meadow_object = $meadow_name->new();
$meadow_object->meadow_options($meadow_options);
$meadow_object->total_running_workers_limit($local_cpus) if($meadow_object->can('total_running_workers_limit'));
} else {
die "Meadow '$meadow_name' does not seem to be available on this machine, please investigate"; die "Meadow '$meadow_name' does not seem to be available on this machine, please investigate";
} }
warn "Current meadow: '$meadow_name'\n";
my $meadow_object = $meadow_name->new();
$meadow_object->meadow_options($meadow_options);
$meadow_object->total_running_workers_max($total_workers_max) if($total_workers_max);
$meadow_object->pending_adjust(not $no_pend_adjust); $meadow_object->pending_adjust(not $no_pend_adjust);
if($self->{'run_job_id'}) { if($self->{'run_job_id'}) {
$worker_limit = 1; $submit_workers_max = 1;
} }
$meadow_object->submitted_workers_limit($worker_limit); $meadow_object->submit_workers_max($submit_workers_max);
$meadow_object->pipeline_name($pipeline_name); $meadow_object->pipeline_name($pipeline_name);
if($reset_job_id) { $queen->reset_and_fetch_job_by_dbID($reset_job_id); } if($reset_job_id) { $queen->reset_and_fetch_job_by_dbID($reset_job_id); }
...@@ -465,7 +464,7 @@ __DATA__ ...@@ -465,7 +464,7 @@ __DATA__
and to send the requested number of workers to open machines via the runWorker.pl script. and to send the requested number of workers to open machines via the runWorker.pl script.
It is also responsible for interfacing with the Queen to identify workers which died It is also responsible for interfacing with the Queen to identify workers which died
unexpectantly so that she can free the dead workers and reclaim unfinished jobs. unexpectedly so that she can free the dead workers and reclaim unfinished jobs.
=head1 USAGE EXAMPLES =head1 USAGE EXAMPLES
...@@ -478,8 +477,8 @@ __DATA__ ...@@ -478,8 +477,8 @@ __DATA__
# Do not run any additional Workers, just check for the current status of the pipeline: # Do not run any additional Workers, just check for the current status of the pipeline:
beekeeper.pl -url mysql://username:secret@hostname:port/ehive_dbname beekeeper.pl -url mysql://username:secret@hostname:port/ehive_dbname
# Run the pipeline in automatic mode (-loop), run all the workers locally (-local) and allow for 3 parallel workers (-local_cpus 3) # Run the pipeline in automatic mode (-loop), run all the workers locally (-meadow_name LOCAL) and allow for 3 parallel workers (-total_workers_max 3)
beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -local -local_cpus 3 -loop beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -meadow_name LOCAL -total_workers_max 3 -loop
# Run in automatic mode, but only restrict to running the 'fast_blast' analysis # Run in automatic mode, but only restrict to running the 'fast_blast' analysis
beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -logic_name fast_blast -loop beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -logic_name fast_blast -loop
...@@ -518,9 +517,8 @@ __DATA__ ...@@ -518,9 +517,8 @@ __DATA__
=head2 Meadow control =head2 Meadow control
-local : run jobs on local CPU (fork) -total_workers_max <num> : max # workers to be running in parallel
-local_cpus <num> : max # workers to be running locally -submit_workers_max <num> : max # workers to create per loop
-wlimit <num> : max # workers to create per loop
-no_pend : don't adjust needed workers by pending workers -no_pend : don't adjust needed workers by pending workers
-meadow_name <string> : the desired Meadow class name, such as 'LSF' or 'LOCAL' -meadow_name <string> : the desired Meadow class name, such as 'LSF' or 'LOCAL'
-meadow_options <string> : passes <string> to the Meadow submission command as <options> (formerly lsf_options) -meadow_options <string> : passes <string> to the Meadow submission command as <options> (formerly lsf_options)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment