Commit 7964fa0d authored by Leo Gordon's avatar Leo Gordon
Browse files

schema_change: introducing role table and Role object

parent 28821307
...@@ -367,9 +367,10 @@ sub reset_or_grab_job_by_dbID { ...@@ -367,9 +367,10 @@ sub reset_or_grab_job_by_dbID {
sub grab_jobs_for_worker { sub grab_jobs_for_worker {
my ($self, $worker, $how_many_this_batch, $workers_rank) = @_; my ($self, $worker, $how_many_this_batch, $workers_rank) = @_;
my $analysis_id = $worker->analysis_id(); my $current_role = $worker->current_role;
my $worker_id = $worker->dbID(); my $analysis_id = $current_role->analysis_id();
my $offset = $how_many_this_batch*$workers_rank; my $worker_id = $worker->dbID();
my $offset = $how_many_this_batch*$workers_rank;
my $prefix_sql = ($self->dbc->driver eq 'mysql') ? qq{ my $prefix_sql = ($self->dbc->driver eq 'mysql') ? qq{
UPDATE job j UPDATE job j
...@@ -431,9 +432,10 @@ sub grab_jobs_for_worker { ...@@ -431,9 +432,10 @@ sub grab_jobs_for_worker {
sub release_undone_jobs_from_worker { sub release_undone_jobs_from_worker {
my ($self, $worker, $msg) = @_; my ($self, $worker, $msg) = @_;
my $max_retry_count = $worker->analysis->max_retry_count(); my $current_role = $worker->current_role;
my $analysis = $current_role->analysis;
my $max_retry_count = $analysis->max_retry_count();
my $worker_id = $worker->dbID(); my $worker_id = $worker->dbID();
my $analysis = $worker->analysis();
#first just reset the claimed jobs, these don't need a retry_count index increment: #first just reset the claimed jobs, these don't need a retry_count index increment:
# (previous worker_id does not matter, because that worker has never had a chance to run the job) # (previous worker_id does not matter, because that worker has never had a chance to run the job)
......
...@@ -184,6 +184,7 @@ our %adaptor_type_2_package_name = ( ...@@ -184,6 +184,7 @@ our %adaptor_type_2_package_name = (
'NakedTable' => 'Bio::EnsEMBL::Hive::DBSQL::NakedTableAdaptor', 'NakedTable' => 'Bio::EnsEMBL::Hive::DBSQL::NakedTableAdaptor',
'ResourceClass' => 'Bio::EnsEMBL::Hive::DBSQL::ResourceClassAdaptor', 'ResourceClass' => 'Bio::EnsEMBL::Hive::DBSQL::ResourceClassAdaptor',
'ResourceDescription' => 'Bio::EnsEMBL::Hive::DBSQL::ResourceDescriptionAdaptor', 'ResourceDescription' => 'Bio::EnsEMBL::Hive::DBSQL::ResourceDescriptionAdaptor',
'Role' => 'Bio::EnsEMBL::Hive::DBSQL::RoleAdaptor',
'Queen' => 'Bio::EnsEMBL::Hive::Queen', 'Queen' => 'Bio::EnsEMBL::Hive::Queen',
# aliases: # aliases:
......
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
$dba->get_LogMessageAdaptor->store_job_message($job_id, $msg, $is_error); $dba->get_LogMessageAdaptor->store_job_message($job_id, $msg, $is_error);
$dba->get_LogMessageAdaptor->store_worker_message($worker_id, $msg, $is_error); $dba->get_LogMessageAdaptor->store_worker_message($worker, $msg, $is_error);
=head1 DESCRIPTION =head1 DESCRIPTION
...@@ -67,7 +67,10 @@ sub store_job_message { ...@@ -67,7 +67,10 @@ sub store_job_message {
sub store_worker_message { sub store_worker_message {
my ($self, $worker_id, $msg, $is_error) = @_; my ($self, $worker, $msg, $is_error) = @_;
my $worker_id = $worker->dbID;
my $role_id = $worker->current_role && $worker->current_role->dbID;
chomp $msg; # we don't want that last "\n" in the database chomp $msg; # we don't want that last "\n" in the database
...@@ -75,12 +78,12 @@ sub store_worker_message { ...@@ -75,12 +78,12 @@ sub store_worker_message {
# Note: the timestamp 'time' column will be set automatically # Note: the timestamp 'time' column will be set automatically
my $sql = qq{ my $sql = qq{
INSERT INTO $table_name (worker_id, status, msg, is_error) INSERT INTO $table_name (worker_id, role_id, status, msg, is_error)
SELECT worker_id, status, ?, ? SELECT worker_id, ?, status, ?, ?
FROM worker WHERE worker_id=? FROM worker WHERE worker_id=?
}; };
my $sth = $self->prepare( $sql ); my $sth = $self->prepare( $sql );
$sth->execute( $msg, $is_error ? 1 : 0, $worker_id ); $sth->execute( $role_id, $msg, $is_error ? 1 : 0, $worker_id );
$sth->finish(); $sth->finish();
} }
......
=pod
=head1 NAME
Bio::EnsEMBL::Hive::DBSQL::RoleAdaptor
=head1 SYNOPSIS
$role_adaptor = $db_adaptor->get_RoleAdaptor;
$role_adaptor = $role_object->adaptor;
=head1 DESCRIPTION
Module to encapsulate all db access for persistent class Role.
There should be just one per application and database connection.
=head1 LICENSE
Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License
is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and limitations under the License.
=head1 CONTACT
Please subscribe to the Hive mailing list: http://listserver.ebi.ac.uk/mailman/listinfo/ehive-users to discuss Hive-related questions or to be notified of our updates
=cut
package Bio::EnsEMBL::Hive::DBSQL::RoleAdaptor;
use strict;
use Bio::EnsEMBL::Hive::Role;
use base ('Bio::EnsEMBL::Hive::DBSQL::ObjectAdaptor');
sub default_table_name {
return 'role';
}
sub default_insertion_method {
return 'INSERT';
}
sub object_class {
return 'Bio::EnsEMBL::Hive::Role';
}
sub finalize_role {
my ($self, $role) = @_;
my $role_id = $role->dbID;
my $when_finished = $role->when_finished ? "'".$role->when_finished."'" : 'CURRENT_TIMESTAMP';
my $sql = "UPDATE role SET when_finished=$when_finished WHERE role_id=$role_id";
$self->dbc->do( $sql );
}
sub fetch_last_by_worker_id {
my ($self, $worker_id) = @_;
return $self->fetch_all( "WHERE worker_id=$worker_id ORDER BY when_started DESC LIMIT 1", 1 );
}
1;
...@@ -70,8 +70,9 @@ use File::Path 'make_path'; ...@@ -70,8 +70,9 @@ use File::Path 'make_path';
use Bio::EnsEMBL::Hive::Utils ('destringify', 'dir_revhash'); # NB: needed by invisible code use Bio::EnsEMBL::Hive::Utils ('destringify', 'dir_revhash'); # NB: needed by invisible code
use Bio::EnsEMBL::Hive::AnalysisJob; use Bio::EnsEMBL::Hive::AnalysisJob;
use Bio::EnsEMBL::Hive::Worker; use Bio::EnsEMBL::Hive::Role;
use Bio::EnsEMBL::Hive::Scheduler; use Bio::EnsEMBL::Hive::Scheduler;
use Bio::EnsEMBL::Hive::Worker;
use base ('Bio::EnsEMBL::Hive::DBSQL::ObjectAdaptor'); use base ('Bio::EnsEMBL::Hive::DBSQL::ObjectAdaptor');
...@@ -283,19 +284,28 @@ sub specialize_new_worker { ...@@ -283,19 +284,28 @@ sub specialize_new_worker {
die "No analysis suitable for the worker was found\n"; die "No analysis suitable for the worker was found\n";
} }
# now set it in the $worker: # TODO: remove setting this in the Worker once it works well via Role:
$worker->analysis_id( $analysis_id ); $worker->analysis_id( $analysis_id );
$self->update_analysis_id( $worker ); # autoloaded $self->update_analysis_id( $worker ); # autoloaded
my $role_adaptor = $self->db->get_RoleAdaptor;
if( my $old_role = $worker->current_role ) {
$role_adaptor->finalize_role( $old_role );
}
my $new_role = Bio::EnsEMBL::Hive::Role->new(
'worker' => $worker,
'analysis_id' => $analysis_id,
);
$role_adaptor->store( $new_role );
$worker->current_role( $new_role );
if($special_batch) { if($special_batch) {
$worker->special_batch( $special_batch ); $worker->special_batch( $special_batch );
} else { # count it as autonomous worker sharing the load of that analysis: } else { # count it as autonomous worker sharing the load of that analysis:
$analysis_stats_adaptor->update_status($analysis_id, 'WORKING'); $analysis_stats_adaptor->update_status($analysis_id, 'WORKING');
$analysis_stats_adaptor->decrease_required_workers($worker->analysis_id); $analysis_stats_adaptor->decrease_required_workers( $new_role->analysis_id );
} }
# The following increment used to be done only when no specific task was given to the worker, # The following increment used to be done only when no specific task was given to the worker,
...@@ -305,7 +315,7 @@ sub specialize_new_worker { ...@@ -305,7 +315,7 @@ sub specialize_new_worker {
# so I am (temporarily?) simplifying the accounting algorithm. # so I am (temporarily?) simplifying the accounting algorithm.
# #
unless( $self->db->hive_use_triggers() ) { unless( $self->db->hive_use_triggers() ) {
$analysis_stats_adaptor->increase_running_workers($worker->analysis_id); $analysis_stats_adaptor->increase_running_workers( $new_role->analysis_id );
} }
} }
...@@ -315,26 +325,33 @@ sub register_worker_death { ...@@ -315,26 +325,33 @@ sub register_worker_death {
return unless($worker); return unless($worker);
my $current_role = $worker->current_role;
my $worker_id = $worker->dbID; my $worker_id = $worker->dbID;
my $work_done = $worker->work_done; my $work_done = $worker->work_done;
my $cause_of_death = $worker->cause_of_death || 'UNKNOWN'; # make sure we do not attempt to insert a void my $cause_of_death = $worker->cause_of_death || 'UNKNOWN'; # make sure we do not attempt to insert a void
my $died = $worker->died; my $worker_died = $worker->died;
if( $current_role ) {
$current_role->when_finished( $worker_died );
$self->db->get_RoleAdaptor->finalize_role( $current_role );
}
my $sql = "UPDATE worker SET status='DEAD', work_done='$work_done', cause_of_death='$cause_of_death'" my $sql = "UPDATE worker SET status='DEAD', work_done='$work_done', cause_of_death='$cause_of_death'"
. ( $self_burial ? ', last_check_in=CURRENT_TIMESTAMP ' : '' ) . ( $self_burial ? ', last_check_in=CURRENT_TIMESTAMP ' : '' )
. ( $died ? ", died='$died'" : ', died=CURRENT_TIMESTAMP' ) . ( $worker_died ? ", died='$worker_died'" : ', died=CURRENT_TIMESTAMP' )
. " WHERE worker_id='$worker_id' "; . " WHERE worker_id='$worker_id' ";
$self->dbc->do( $sql ); $self->dbc->do( $sql );
if(my $analysis_id = $worker->analysis_id) { if( my $analysis_id = $current_role && $current_role->analysis_id ) {
my $analysis_stats_adaptor = $self->db->get_AnalysisStatsAdaptor; my $analysis_stats_adaptor = $self->db->get_AnalysisStatsAdaptor;
unless( $self->db->hive_use_triggers() ) { unless( $self->db->hive_use_triggers() ) {
$analysis_stats_adaptor->decrease_running_workers($worker->analysis_id); $analysis_stats_adaptor->decrease_running_workers( $analysis_id );
} }
unless( $cause_of_death eq 'NO_WORK' unless( $cause_of_death eq 'NO_ROLE'
or $cause_of_death eq 'NO_WORK'
or $cause_of_death eq 'JOB_LIMIT' or $cause_of_death eq 'JOB_LIMIT'
or $cause_of_death eq 'HIVE_OVERLOAD' or $cause_of_death eq 'HIVE_OVERLOAD'
or $cause_of_death eq 'LIFESPAN' or $cause_of_death eq 'LIFESPAN'
...@@ -343,10 +360,10 @@ sub register_worker_death { ...@@ -343,10 +360,10 @@ sub register_worker_death {
} }
# re-sync the analysis_stats when a worker dies as part of dynamic sync system # re-sync the analysis_stats when a worker dies as part of dynamic sync system
if($self->safe_synchronize_AnalysisStats($worker->analysis->stats)->status ne 'DONE') { if($self->safe_synchronize_AnalysisStats( $current_role->analysis->stats )->status ne 'DONE') {
# since I'm dying I should make sure there is someone to take my place after I'm gone ... # since I'm dying I should make sure there is someone to take my place after I'm gone ...
# above synch still sees me as a 'living worker' so I need to compensate for that # above synch still sees me as a 'living worker' so I need to compensate for that
$analysis_stats_adaptor->increase_required_workers($worker->analysis_id); $analysis_stats_adaptor->increase_required_workers( $analysis_id );
} }
} }
} }
...@@ -372,14 +389,14 @@ sub check_for_dead_workers { # scans the whole Valley for lost Workers (but i ...@@ -372,14 +389,14 @@ sub check_for_dead_workers { # scans the whole Valley for lost Workers (but i
$mt_and_pid_to_worker_status{$meadow_type} ||= $meadow->status_of_all_our_workers; # only run this once per reachable Meadow $mt_and_pid_to_worker_status{$meadow_type} ||= $meadow->status_of_all_our_workers; # only run this once per reachable Meadow
my $process_id = $worker->process_id; my $process_id = $worker->process_id;
if(my $status = $mt_and_pid_to_worker_status{$meadow_type}{$process_id}) { # can be RUN|PEND|xSUSP if(my $status = $mt_and_pid_to_worker_status{$meadow_type}{$process_id}) { # can be RUN|PEND|xSUSP
$worker_status_counts{$meadow_type}{$status}++; $worker_status_counts{$meadow_type}{$status}++;
} else { } else {
$worker_status_counts{$meadow_type}{'LOST'}++; $worker_status_counts{$meadow_type}{'LOST'}++;
$mt_and_pid_to_lost_worker{$meadow_type}{$process_id} = $worker; $mt_and_pid_to_lost_worker{$meadow_type}{$process_id} = $worker;
} }
} else { } else {
$worker_status_counts{$meadow_type}{'UNREACHABLE'}++; # Worker is unreachable from this Valley $worker_status_counts{$meadow_type}{'UNREACHABLE'}++; # Worker is unreachable from this Valley
} }
...@@ -390,6 +407,8 @@ sub check_for_dead_workers { # scans the whole Valley for lost Workers (but i ...@@ -390,6 +407,8 @@ sub check_for_dead_workers { # scans the whole Valley for lost Workers (but i
warn "GarbageCollector:\t[$meadow_type Meadow:]\t".join(', ', map { "$_:$worker_status_counts{$meadow_type}{$_}" } keys %{$worker_status_counts{$meadow_type}})."\n\n"; warn "GarbageCollector:\t[$meadow_type Meadow:]\t".join(', ', map { "$_:$worker_status_counts{$meadow_type}{$_}" } keys %{$worker_status_counts{$meadow_type}})."\n\n";
} }
my $role_adaptor = $self->db->get_RoleAdaptor;
while(my ($meadow_type, $pid_to_lost_worker) = each %mt_and_pid_to_lost_worker) { while(my ($meadow_type, $pid_to_lost_worker) = each %mt_and_pid_to_lost_worker) {
my $this_meadow = $valley->available_meadow_hash->{$meadow_type}; my $this_meadow = $valley->available_meadow_hash->{$meadow_type};
...@@ -413,6 +432,7 @@ sub check_for_dead_workers { # scans the whole Valley for lost Workers (but i ...@@ -413,6 +432,7 @@ sub check_for_dead_workers { # scans the whole Valley for lost Workers (but i
while(my ($process_id, $worker) = each %$pid_to_lost_worker) { while(my ($process_id, $worker) = each %$pid_to_lost_worker) {
$worker->died( $report_entries->{$process_id}{'died'} ); $worker->died( $report_entries->{$process_id}{'died'} );
$worker->cause_of_death( $report_entries->{$process_id}{'cause_of_death'} ); $worker->cause_of_death( $report_entries->{$process_id}{'cause_of_death'} );
$worker->current_role( $role_adaptor->fetch_last_by_worker_id( $worker->dbID ) );
$self->register_worker_death( $worker ); $self->register_worker_death( $worker );
} }
...@@ -675,7 +695,7 @@ sub count_running_workers { ...@@ -675,7 +695,7 @@ sub count_running_workers {
sub get_workers_rank { sub get_workers_rank {
my ($self, $worker) = @_; my ($self, $worker) = @_;
return $self->count_all( "status!='DEAD' AND analysis_id=".$worker->analysis_id." AND worker_id<".$worker->dbID ); return $self->count_all( "status!='DEAD' AND analysis_id=" . $worker->current_role->analysis_id . " AND worker_id<" . $worker->dbID );
} }
......
=pod
=head1 NAME
Bio::EnsEMBL::Hive::Role
=head1 DESCRIPTION
Role is a state of a Worker while performing jobs of a particular Analysis.
=head1 LICENSE
Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License
is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and limitations under the License.
=head1 CONTACT
Please subscribe to the Hive mailing list: http://listserver.ebi.ac.uk/mailman/listinfo/ehive-users to discuss Hive-related questions or to be notified of our updates
=head1 APPENDIX
The rest of the documentation details each of the object methods.
Internal methods are usually preceded with a _
=cut
package Bio::EnsEMBL::Hive::Role;
use strict;
use base ( 'Bio::EnsEMBL::Hive::Storable' );
=head1 AUTOLOADED
worker_id / worker
analysis_id / analysis
=cut
sub when_started {
my $self = shift;
$self->{'_when_started'} = shift if(@_);
return $self->{'_when_started'};
}
sub when_finished {
my $self = shift;
$self->{'_when_finished'} = shift if(@_);
return $self->{'_when_finished'};
}
sub attempted_jobs {
my $self = shift;
$self->{'_attempted_jobs'} = shift if(@_);
return $self->{'_attempted_jobs'} || 0;
}
sub done_jobs {
my $self = shift;
$self->{'_done_jobs'} = shift if(@_);
return $self->{'_done_jobs'} || 0;
}
sub register_attempt {
my $self = shift;
my $success = shift;
$self->{'_attempted_jobs'}++;
$self->{'_done_jobs'} += $success;
if( my $adaptor = $self->adaptor ) {
$adaptor->update_attempted_jobs_AND_done_jobs( $self );
}
}
1;
...@@ -94,7 +94,7 @@ sub fetch_input { ...@@ -94,7 +94,7 @@ sub fetch_input {
$self->param('ignores', \@ignores); $self->param('ignores', \@ignores);
# Would be good to have this from eHive # Would be good to have this from eHive
my @ehive_tables = qw(hive_meta pipeline_wide_parameters worker dataflow_rule analysis_base analysis_ctrl_rule job accu log_message job_file analysis_data resource_description analysis_stats analysis_stats_monitor msg progress resource_class); my @ehive_tables = qw(hive_meta pipeline_wide_parameters worker dataflow_rule analysis_base analysis_ctrl_rule job accu log_message job_file analysis_data resource_description analysis_stats analysis_stats_monitor role msg progress resource_class);
$self->param('nb_ehive_tables', scalar(@ehive_tables)); $self->param('nb_ehive_tables', scalar(@ehive_tables));
# Connection parameters # Connection parameters
......
...@@ -210,6 +210,13 @@ sub log_dir { ...@@ -210,6 +210,13 @@ sub log_dir {
## Non-Storable attributes: ## Non-Storable attributes:
sub current_role {
my $self = shift;
$self->{'_current_role'} = shift if(@_);
return $self->{'_current_role'};
}
sub debug { sub debug {
my $self = shift; my $self = shift;
$self->{'_debug'} = shift if(@_); $self->{'_debug'} = shift if(@_);
...@@ -404,9 +411,12 @@ sub get_stderr_redirector { ...@@ -404,9 +411,12 @@ sub get_stderr_redirector {
sub worker_say { sub worker_say {
my ($self, $msg) = @_; my ($self, $msg) = @_;
my $worker_id = $self->dbID(); my $worker_id = $self->dbID();
my $analysis_name = $self->analysis_id ? $self->analysis->logic_name.'('.$self->analysis_id.')' : ''; my $current_role = $self->current_role;
print "Worker $worker_id [ $analysis_name ] $msg\n"; print "Worker $worker_id [ ". ( $current_role
? ('Role '.$current_role->dbID.' , '.$current_role->analysis->logic_name.'('.$current_role->analysis_id.')')
: 'UNSPECIALIZED'
)." ] $msg\n";
} }
...@@ -414,12 +424,12 @@ sub toString { ...@@ -414,12 +424,12 @@ sub toString {
my $self = shift @_; my $self = shift @_;
return join(', ', return join(', ',
'analysis='.($self->analysis_id ? $self->analysis->logic_name.'('.$self->analysis_id.')' : 'UNSPECIALIZED'), 'analysis='.($self->current_role ? $self->current_role->analysis->logic_name.'('.$self->current_role->analysis_id.')' : 'UNSPECIALIZED'),
'resource_class_id='.($self->resource_class_id || 'NULL'), 'resource_class_id='.($self->resource_class_id || 'NULL'),
'meadow='.$self->meadow_type.'/'.$self->meadow_name, 'meadow='.$self->meadow_type.'/'.$self->meadow_name,
'process='.$self->process_id.'@'.$self->host, 'process='.$self->process_id.'@'.$self->host,
'last_check_in='.$self->last_check_in, 'last_check_in='.$self->last_check_in,
'batch_size='.($self->analysis_id ? $self->analysis->stats->get_or_estimate_batch_size() : 'UNSPECIALIZED'), 'batch_size='.($self->current_role ? $self->current_role->analysis->stats->get_or_estimate_batch_size() : 'UNSPECIALIZED'),
'job_limit='.($self->job_limiter->available_capacity() || 'NONE'), 'job_limit='.($self->job_limiter->available_capacity() || 'NONE'),
'life_span='.($self->life_span || 'UNLIM'), 'life_span='.($self->life_span || 'UNLIM'),
'worker_log_dir='.($self->log_dir || 'STDOUT/STDERR'), 'worker_log_dir='.($self->log_dir || 'STDOUT/STDERR'),
...@@ -499,7 +509,7 @@ sub run { ...@@ -499,7 +509,7 @@ sub run {
$self->cause_of_death('LIFESPAN'); $self->cause_of_death('LIFESPAN');
} else { } else {
my $desired_batch_size = $self->analysis->stats->get_or_estimate_batch_size(); my $desired_batch_size = $self->current_role->analysis->stats->get_or_estimate_batch_size();
$desired_batch_size = $self->job_limiter->preliminary_offer( $desired_batch_size ); $desired_batch_size = $self->job_limiter->preliminary_offer( $desired_batch_size );
my $workers_rank = $self->adaptor->get_workers_rank( $self ); my $workers_rank = $self->adaptor->get_workers_rank( $self );
...@@ -521,7 +531,7 @@ sub run { ...@@ -521,7 +531,7 @@ sub run {
if($jobs_done_by_batches_loop) { if($jobs_done_by_batches_loop) {
$self->adaptor->db->get_AnalysisStatsAdaptor->interval_update_work_done( $self->adaptor->db->get_AnalysisStatsAdaptor->interval_update_work_done(
$self->analysis->dbID, $self->current_role->analysis->dbID,
$jobs_done_by_batches_loop, $jobs_done_by_batches_loop,
$batches_stopwatch->get_elapsed, $batches_stopwatch->get_elapsed,
$self->{'_interval_partial_timing'}{'FETCH_INPUT'} || 0, $self->{'_interval_partial_timing'}{'FETCH_INPUT'} || 0,
...@@ -532,9 +542,10 @@ sub run { ...@@ -532,9 +542,10 @@ sub run {
# A mechanism whereby workers can be caused to exit even if they were doing fine: # A mechanism whereby workers can be caused to exit even if they were doing fine:
if (!$self->cause_of_death) { if (!$self->cause_of_death) {