Commit 6cd6c078 authored by Jessica Severin's avatar Jessica Severin
Browse files

modified analysis_job_file table to better track job diagnostics

added columns hive_id and retry.  Allows user to join to failed workers
in the hive table, and to see which retry level the job was at when the
STDOUT/STDERR files were generated.  Sets at beginning of job run, and
deletes those for 'empty' files at job end.
parent b0b16a74
......@@ -417,17 +417,25 @@ sub reclaim_job {
sub store_out_files {
my ($self,$job) = @_;
return unless($job and ($job->stdout_file or $job->stderr_file));
return unless($job);
my $sql = "INSERT ignore INTO analysis_job_file (analysis_job_id, type, path) VALUES ";
$sql .= " (" . $job->dbID. ", 'STDOUT', '". $job->stdout_file."')" if($job->stdout_file);
my $sql = sprintf("DELETE from analysis_job_file WHERE hive_id=%d and analysis_job_id=%d",
$job->hive_id, $job->dbID);
$self->dbc->do($sql);
return unless($job->stdout_file or $job->stderr_file);
$sql = "INSERT ignore INTO analysis_job_file (analysis_job_id, hive_id, retry, type, path) VALUES ";
if($job->stdout_file) {
$sql .= sprintf("(%d,%d,%d,'STDOUT','%s')", $job->dbID, $job->hive_id,
$job->retry_count, $job->stdout_file);
}
$sql .= "," if($job->stdout_file and $job->stderr_file);
$sql .= " (" . $job->dbID. ", 'STDERR', '". $job->stderr_file."')" if($job->stderr_file);
#print("$sql\n");
my $sth = $self->prepare($sql);
$sth->execute();
$sth->finish;
if($job->stderr_file) {
$sql .= sprintf("(%d,%d,%d,'STDERR','%s')", $job->dbID, $job->hive_id,
$job->retry_count, $job->stderr_file);
}
$self->dbc->do($sql);
}
......
......@@ -147,15 +147,19 @@ CREATE TABLE analysis_job (
--
-- semantics:
-- analysis_job_id - foreign key
-- hive_id - link to hive table to define which worker claimed this job
-- retry - copy of retry_count of job as it was run
-- type - type of file e.g. STDOUT, STDERR, TMPDIR, ...
-- path - path to file or directory
CREATE TABLE analysis_job_file (
analysis_job_id int(10) NOT NULL,
hive_id int(10) NOT NULL,
retry int(10) NOT NULL,
type varchar(16) NOT NULL default '',
path varchar(255) NOT NULL,
UNIQUE KEY (analysis_job_id, type)
UNIQUE KEY job_hive_type (analysis_job_id, hive_id, type)
);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment