Commit 27403dda authored by Jessica Severin's avatar Jessica Severin
Browse files

added hive_id index to analysis_job table to help with dead_worker

job reseting.  This allowed direct UPDATE..WHERE.. sql to be used.
Also changed the retry_count system: retry_count is only incremented
for jobs that failed (status in ('GET_INPUT','RUN','WRITE_OUTPUT')).
Job that were CLAIMED by the dead worker are just reset without
incrementing the retry_count since they were never attempted to run.
Also the fetching of claimed jobs now has an 'ORDER BY retry_count'
so that jobs that have failed are at the bottom of the list of jobs
to process.  This allows the 'bad' jobs to filter themselves out.
parent ba5578d5
......@@ -290,7 +290,7 @@ sub _default_where_clause {
sub _final_clause {
my $self = shift;
return '';
return 'ORDER BY retry_count';
}
......@@ -378,29 +378,30 @@ sub reset_dead_jobs_for_worker {
my $worker = shift;
throw("must define worker") unless($worker);
#added hive_id index to analysis_job table which made this operation much faster
my ($sql, $sth);
#first just reset the claimed jobs, these don't need a retry_count index increment
$sql = "UPDATE analysis_job SET job_claim='', hive_id=0, status='READY'".
" WHERE status='CLAIMED'".
" AND hive_id='" . $worker->hive_id ."'";
$sth = $self->prepare($sql);
$sth->execute();
$sth->finish;
#print(" done update CLAIMED\n");
# an update with select on status and hive_id took 4seconds per worker to complete,
# while doing a select followed by update on analysis_job_id returned almost instantly
my $sql = "select analysis_job_id from analysis_job ".
" WHERE status in ('CLAIMED','GET_INPUT','RUN','WRITE_OUTPUT')".
" AND hive_id='" . $worker->hive_id ."'";
$sql = "UPDATE analysis_job SET job_claim='', hive_id=0, status='READY'".
" ,retry_count=retry_count+1".
" WHERE status in ('GET_INPUT','RUN','WRITE_OUTPUT')".
" AND hive_id='" . $worker->hive_id ."'";
#print("$sql\n");
my $sth = $self->prepare($sql);
$sth = $self->prepare($sql);
$sth->execute();
my @jobIDS;
while (my ($job_id)=$sth->fetchrow_array()) { push @jobIDS, $job_id; }
$sth->finish;
#print("reset ", scalar(@jobIDS), " jobs\n");
foreach my $job_id (@jobIDS) {
my $sql = "UPDATE analysis_job SET job_claim='', hive_id=0, status='READY'".
" ,retry_count=retry_count+1".
" WHERE analysis_job_id=$job_id";
my $sth = $self->prepare($sql);
$sth->execute();
$sth->finish;
}
#print(" done update BROKEN jobs\n");
}
......
......@@ -161,7 +161,8 @@ CREATE TABLE analysis_job (
PRIMARY KEY (analysis_job_id),
UNIQUE KEY input_id_analysis (input_id, analysis_id),
INDEX job_claim_analysis (job_claim, analysis_id),
INDEX job_status_analysis (status, analysis_id)
INDEX job_status_analysis (status, analysis_id),
INDEX hive_id (hive_id)
);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment