bugfix [to ENSCOMPARASW-136] : life_cycle() now performs autoflow for jobs...

bugfix [to ENSCOMPARASW-136] : life_cycle() now performs autoflow for jobs that choose to exit via incomplete(0)-and-die

modules/Bio/EnsEMBL/Hive/Process.pm

@@ -120,40 +120,59 @@ sub life_cycle {
     my $partial_stopwatch = Bio::EnsEMBL::Hive::Utils::Stopwatch->new();
     my %job_partial_timing = ();
 
+    $job->incomplete(1);    # reinforce, in case the life_cycle is not run by a Worker
     $job->autoflow(1);
 
-    if( $self->can('pre_cleanup') and $job->retry_count()>0 ) {
-        $self->enter_status('PRE_CLEANUP');
-        $self->pre_cleanup;
-    }
-
-    $self->enter_status('FETCH_INPUT');
-    $partial_stopwatch->restart();
-    $self->fetch_input;
-    $job_partial_timing{'FETCH_INPUT'} = $partial_stopwatch->get_elapsed();
+    eval {
+        if( $self->can('pre_cleanup') and $job->retry_count()>0 ) {
+            $self->enter_status('PRE_CLEANUP');
+            $self->pre_cleanup;
+        }
 
-    $self->enter_status('RUN');
-    $partial_stopwatch->restart();
-    $self->run;
-    $job_partial_timing{'RUN'} = $partial_stopwatch->get_elapsed();
+        $self->enter_status('FETCH_INPUT');
+        $partial_stopwatch->restart();
+        $self->fetch_input;
+        $job_partial_timing{'FETCH_INPUT'} = $partial_stopwatch->get_elapsed();
 
-    if($self->execute_writes) {
-        $self->enter_status('WRITE_OUTPUT');
+        $self->enter_status('RUN');
         $partial_stopwatch->restart();
-        $self->write_output;
-        $job_partial_timing{'WRITE_OUTPUT'} = $partial_stopwatch->get_elapsed();
+        $self->run;
+        $job_partial_timing{'RUN'} = $partial_stopwatch->get_elapsed();
+
+        if($self->execute_writes) {
+            $self->enter_status('WRITE_OUTPUT');
+            $partial_stopwatch->restart();
+            $self->write_output;
+            $job_partial_timing{'WRITE_OUTPUT'} = $partial_stopwatch->get_elapsed();
+        } else {
+            print STDERR "\n!!! *no* WRITE_OUTPUT requested, so there will be no AUTOFLOW\n" if($self->debug); 
+        }
+    };
+
+    my $error_msg = $@;
+
+    if( $self->can('post_cleanup') ) {   # may be run to clean up memory even after partially failed attempts
+        eval {
+            $self->enter_status('POST_CLEANUP');
+            $self->post_cleanup;
+        };
+        if($@) {
+            $error_msg .= $@;
+            $job->incomplete(1);
+        }
+    }
 
-        if( $job->autoflow ) {
-            print STDERR "\njob ".$job->dbID." : AUTOFLOW input->output\n" if($self->debug);
-            $job->dataflow_output_id();
+    if( $error_msg ) {
+        if( $job->incomplete ) {    # retransmit the death message if it was not a suicide, continue otherwise
+            die $error_msg;
+        } else {
+            $self->warning( $error_msg );
         }
-    } else {
-        print STDERR "\n!!! *no* WRITE_OUTPUT requested, so there will be no AUTOFLOW\n" if($self->debug); 
     }
 
-    if( $self->can('post_cleanup') ) {   # Todo: may need to run it after the eval, to clean up the memory even after partially failed attempts?
-        $self->enter_status('POST_CLEANUP');
-        $self->post_cleanup;
+    if( $self->execute_writes and $job->autoflow ) {    # AUTOFLOW doesn't have its own status so will have whatever previous state of the job
+        print STDERR "\njob ".$job->dbID." : AUTOFLOW input->output\n" if($self->debug);
+        $job->dataflow_output_id();
     }
 
     my @zombie_funnel_dataflow_rule_ids = keys %{$job->fan_cache};

modules/Bio/EnsEMBL/Hive/Worker.pm

@@ -696,10 +696,9 @@ sub run_one_batch {
 
         my $job_completion_line = "Job $job_id : complete";
 
-        if($msg_thrown) {   # record the message - whether it was a success or failure:
+        if($msg_thrown) {   # record the death message
             my $job_status_at_the_moment = $job->status();
-            my $action = $job->incomplete ? 'died' : 'exited';
-            $job_completion_line = "Job $job_id : $action in status '$job_status_at_the_moment' for the following reason: $msg_thrown";
+            $job_completion_line = "Job $job_id : died in status '$job_status_at_the_moment' for the following reason: $msg_thrown";
             $self->adaptor->db->get_LogMessageAdaptor()->store_job_message($job_id, $msg_thrown, $job->incomplete );
         }