Commit 6cd6c078 authored by Jessica Severin's avatar Jessica Severin
Browse files

modified analysis_job_file table to better track job diagnostics

added columns hive_id and retry.  Allows user to join to failed workers
in the hive table, and to see which retry level the job was at when the
STDOUT/STDERR files were generated.  Sets at beginning of job run, and
deletes those for 'empty' files at job end.
parent b0b16a74
...@@ -417,17 +417,25 @@ sub reclaim_job { ...@@ -417,17 +417,25 @@ sub reclaim_job {
sub store_out_files { sub store_out_files {
my ($self,$job) = @_; my ($self,$job) = @_;
return unless($job and ($job->stdout_file or $job->stderr_file)); return unless($job);
my $sql = "INSERT ignore INTO analysis_job_file (analysis_job_id, type, path) VALUES "; my $sql = sprintf("DELETE from analysis_job_file WHERE hive_id=%d and analysis_job_id=%d",
$sql .= " (" . $job->dbID. ", 'STDOUT', '". $job->stdout_file."')" if($job->stdout_file); $job->hive_id, $job->dbID);
$self->dbc->do($sql);
return unless($job->stdout_file or $job->stderr_file);
$sql = "INSERT ignore INTO analysis_job_file (analysis_job_id, hive_id, retry, type, path) VALUES ";
if($job->stdout_file) {
$sql .= sprintf("(%d,%d,%d,'STDOUT','%s')", $job->dbID, $job->hive_id,
$job->retry_count, $job->stdout_file);
}
$sql .= "," if($job->stdout_file and $job->stderr_file); $sql .= "," if($job->stdout_file and $job->stderr_file);
$sql .= " (" . $job->dbID. ", 'STDERR', '". $job->stderr_file."')" if($job->stderr_file); if($job->stderr_file) {
#print("$sql\n"); $sql .= sprintf("(%d,%d,%d,'STDERR','%s')", $job->dbID, $job->hive_id,
$job->retry_count, $job->stderr_file);
}
my $sth = $self->prepare($sql); $self->dbc->do($sql);
$sth->execute();
$sth->finish;
} }
......
...@@ -147,15 +147,19 @@ CREATE TABLE analysis_job ( ...@@ -147,15 +147,19 @@ CREATE TABLE analysis_job (
-- --
-- semantics: -- semantics:
-- analysis_job_id - foreign key -- analysis_job_id - foreign key
-- hive_id - link to hive table to define which worker claimed this job
-- retry - copy of retry_count of job as it was run
-- type - type of file e.g. STDOUT, STDERR, TMPDIR, ... -- type - type of file e.g. STDOUT, STDERR, TMPDIR, ...
-- path - path to file or directory -- path - path to file or directory
CREATE TABLE analysis_job_file ( CREATE TABLE analysis_job_file (
analysis_job_id int(10) NOT NULL, analysis_job_id int(10) NOT NULL,
hive_id int(10) NOT NULL,
retry int(10) NOT NULL,
type varchar(16) NOT NULL default '', type varchar(16) NOT NULL default '',
path varchar(255) NOT NULL, path varchar(255) NOT NULL,
UNIQUE KEY (analysis_job_id, type) UNIQUE KEY job_hive_type (analysis_job_id, hive_id, type)
); );
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment