added FAILED status to annalysis_job which is set when a job is reset

on failure and retry_count>=5. Also changed Queen analysis summary to
classify an analysis as 'DONE' when all jobs are either DONE or FAILED
and hence allow the processing to proceed forward.

modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisJobAdaptor.pm

@@ -396,11 +396,23 @@ sub reset_dead_jobs_for_worker {
   $sql = "UPDATE analysis_job SET job_claim='', hive_id=0, status='READY'".
          " ,retry_count=retry_count+1".
          " WHERE status in ('GET_INPUT','RUN','WRITE_OUTPUT')".
+	 " AND retry_count<5".
          " AND hive_id='" . $worker->hive_id ."'";
   #print("$sql\n");
   $sth = $self->prepare($sql);
   $sth->execute();
   $sth->finish;
+
+  $sql = "UPDATE analysis_job SET status='FAILED'".
+         " ,retry_count=retry_count+1".
+         " WHERE status in ('GET_INPUT','RUN','WRITE_OUTPUT')".
+	 " AND retry_count>=5".
+         " AND hive_id='" . $worker->hive_id ."'";
+  #print("$sql\n");
+  $sth = $self->prepare($sql);
+  $sth->execute();
+  $sth->finish;
+
   #print(" done update BROKEN jobs\n");
 }
 

modules/Bio/EnsEMBL/Hive/Queen.pm

@@ -94,7 +94,7 @@ sub create_new_worker {
   my $analStatsDBA = $self->db->get_AnalysisStatsAdaptor;
   return undef unless($analStatsDBA);
 
-  return undef if($self->get_hive_current_load() >= 1.0);
+  return undef if($self->get_hive_current_load() >= 1.5);
   
   unless($analysis_id) {
     my ($anal_stats) = @{$analStatsDBA->fetch_by_needed_workers(1)};
@@ -229,7 +229,10 @@ sub update_analysis_stats {
       }
       $analysisStats->num_required_workers($numWorkers);
     }
-    if($status eq 'DONE') { $analysisStats->done_job_count($count); }
+    if(($status eq 'DONE') or ($status eq 'FAILED')) { 
+      $count += $analysisStats->done_job_count(); 
+      $analysisStats->done_job_count($count); 
+    }
   }
   $analysisStats->determine_status()->update() if($analysisStats);
   $sth->finish;