added hive_id index to analysis_job table to help with dead_worker

job reseting.  This allowed direct UPDATE..WHERE.. sql to be used.
Also changed the retry_count system: retry_count is only incremented
for jobs that failed (status in ('GET_INPUT','RUN','WRITE_OUTPUT')).
Job that were CLAIMED by the dead worker are just reset without
incrementing the retry_count since they were never attempted to run.
Also the fetching of claimed jobs now has an 'ORDER BY retry_count'
so that jobs that have failed are at the bottom of the list of jobs
to process.  This allows the 'bad' jobs to filter themselves out.

modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisJobAdaptor.pm

@@ -290,7 +290,7 @@ sub _default_where_clause {
 
 sub _final_clause {
   my $self = shift;
-  return '';
+  return 'ORDER BY retry_count';
 }
 
 
@@ -378,29 +378,30 @@ sub reset_dead_jobs_for_worker {
   my $worker = shift;
   throw("must define worker") unless($worker);
 
+  #added hive_id index to analysis_job table which made this operation much faster
+
+  my ($sql, $sth);
+  #first just reset the claimed jobs, these don't need a retry_count index increment
+  $sql = "UPDATE analysis_job SET job_claim='', hive_id=0, status='READY'".
+         " WHERE status='CLAIMED'".
+         " AND hive_id='" . $worker->hive_id ."'";
+  $sth = $self->prepare($sql);
+  $sth->execute();
+  $sth->finish;
+  #print("  done update CLAIMED\n");
+
   # an update with select on status and hive_id took 4seconds per worker to complete,
   # while doing a select followed by update on analysis_job_id returned almost instantly
   
-  my $sql = "select analysis_job_id from analysis_job ".
-            " WHERE status in ('CLAIMED','GET_INPUT','RUN','WRITE_OUTPUT')".
-            " AND hive_id='" . $worker->hive_id ."'";
-
+  $sql = "UPDATE analysis_job SET job_claim='', hive_id=0, status='READY'".
+         " ,retry_count=retry_count+1".
+         " WHERE status in ('GET_INPUT','RUN','WRITE_OUTPUT')".
+         " AND hive_id='" . $worker->hive_id ."'";
   #print("$sql\n");
-  my $sth = $self->prepare($sql);
+  $sth = $self->prepare($sql);
   $sth->execute();
-  my @jobIDS;
-  while (my ($job_id)=$sth->fetchrow_array()) { push @jobIDS, $job_id; }
   $sth->finish;
-  #print("reset ", scalar(@jobIDS), " jobs\n");
-
-  foreach my $job_id (@jobIDS) {
-    my $sql = "UPDATE analysis_job SET job_claim='', hive_id=0, status='READY'".
-              " ,retry_count=retry_count+1".
-              " WHERE analysis_job_id=$job_id";
-    my $sth = $self->prepare($sql);
-    $sth->execute();
-    $sth->finish;
-  }          
+  #print(" done update BROKEN jobs\n");
 }
 
 

sql/tables.sql

@@ -161,7 +161,8 @@ CREATE TABLE analysis_job (
   PRIMARY KEY                  (analysis_job_id),
   UNIQUE KEY input_id_analysis (input_id, analysis_id),
   INDEX job_claim_analysis     (job_claim, analysis_id),
-  INDEX job_status_analysis    (status, analysis_id)
+  INDEX job_status_analysis    (status, analysis_id),
+  INDEX hive_id                (hive_id)
 );