From 4c6b44e4baefffa869903cc0f8a1315d40070529 Mon Sep 17 00:00:00 2001
From: Leo Gordon <lg4@ebi.ac.uk>
Date: Fri, 9 Nov 2012 12:05:28 +0000
Subject: [PATCH] bugfix: make sure we release claimed jobs from a manually-run
worker whose Runnable fails at compilation (thanks, Miguel!)
---
modules/Bio/EnsEMBL/Hive/Queen.pm | 3 ++-
modules/Bio/EnsEMBL/Hive/Worker.pm | 33 +++++++++++++++++++-----------
2 files changed, 23 insertions(+), 13 deletions(-)
diff --git a/modules/Bio/EnsEMBL/Hive/Queen.pm b/modules/Bio/EnsEMBL/Hive/Queen.pm
index 2beb6525b..fe9c53e40 100644
--- a/modules/Bio/EnsEMBL/Hive/Queen.pm
+++ b/modules/Bio/EnsEMBL/Hive/Queen.pm
@@ -319,7 +319,8 @@ sub register_worker_death {
} elsif($cod eq 'UNKNOWN'
or $cod eq 'MEMLIMIT'
or $cod eq 'RUNLIMIT'
- or $cod eq 'KILLED_BY_USER') {
+ or $cod eq 'KILLED_BY_USER'
+ or $cod eq 'CONTAMINATED') {
$self->db->get_AnalysisJobAdaptor->release_undone_jobs_from_worker($worker);
}
diff --git a/modules/Bio/EnsEMBL/Hive/Worker.pm b/modules/Bio/EnsEMBL/Hive/Worker.pm
index ae387720a..1cfd77c0d 100644
--- a/modules/Bio/EnsEMBL/Hive/Worker.pm
+++ b/modules/Bio/EnsEMBL/Hive/Worker.pm
@@ -519,15 +519,23 @@ sub run {
$self->print_worker();
if( $self->compile_module_once() ) {
- $self->enter_status('COMPILATION');
- my $runnable_object = $self->analysis->process or die "Unknown compilation error";
- $runnable_object->db( $self->adaptor->db );
- $runnable_object->worker( $self );
- $runnable_object->debug( $self->debug );
- $runnable_object->execute_writes( $self->execute_writes );
-
- $self->runnable_object( $runnable_object );
- $self->enter_status('READY');
+ eval {
+ $self->enter_status('COMPILATION');
+ my $runnable_object = $self->analysis->process or die "Unknown compilation error";
+ $runnable_object->db( $self->adaptor->db );
+ $runnable_object->worker( $self );
+ $runnable_object->debug( $self->debug );
+ $runnable_object->execute_writes( $self->execute_writes );
+
+ $self->runnable_object( $runnable_object );
+ $self->enter_status('READY');
+ 1;
+ } or do {
+ my $msg = "Could not compile Runnable '".$self->analysis->module."' :\n\t".$@;
+ warn "$msg\n";
+
+ $self->cause_of_death('CONTAMINATED');
+ }
}
$self->adaptor->db->dbc->disconnect_when_inactive(0);
@@ -535,7 +543,8 @@ sub run {
my $min_batch_time = $self->analysis->stats->min_batch_time();
my $job_adaptor = $self->adaptor->db->get_AnalysisJobAdaptor;
- do { # Worker's lifespan loop (ends only when the worker dies)
+ while (!$self->cause_of_death) { # Worker's lifespan loop (ends only when the worker dies for any reason)
+
my $batches_stopwatch = Bio::EnsEMBL::Hive::Utils::Stopwatch->new()->restart();
my $jobs_done_by_batches_loop = 0; # by all iterations of internal loop
$self->{'_interval_partial_timing'} = {};
@@ -591,7 +600,7 @@ sub run {
$self->cause_of_death('HIVE_OVERLOAD');
}
- } while (!$self->cause_of_death); # /Worker's lifespan loop
+ } # /Worker's lifespan loop
# have runnable clean up any global/process files/data it may have created
if($self->perform_cleanup) {
@@ -609,7 +618,7 @@ sub run {
printf("dbc %d disconnect cycles\n", $self->adaptor->db->dbc->disconnect_count);
print("total jobs completed : ", $self->work_done, "\n");
- if( $self->log_dir() ) {
+ if( $self->log_dir ) {
$self->get_stdout_redirector->pop();
$self->get_stderr_redirector->pop();
}
--
GitLab