Skip to content

GitLab

Commit 160244e9 authored by Leo Gordon's avatar Leo Gordon
Browse files

schema_change: renamed TIMESTAMPed columns to have when_ prefix and added worker.when_seen

parent 6b6a4997
Hide whitespace changes
Inline Side-by-side
Showing with 173 additions and 111 deletions
modules/Bio/EnsEMBL/Hive/AnalysisJob.pm
View file @ 160244e9
......@@ -101,10 +101,10 @@ sub retry_count {
return $self->{'_retry_count'};
}
sub completed {
sub when_completed {
my $self = shift;
$self->{'_completed'} = shift if(@_);
return $self->{'_completed'};
$self->{'_when_completed'} = shift if(@_);
return $self->{'_when_completed'};
}
sub runtime_msec {
......
modules/Bio/EnsEMBL/Hive/AnalysisStats.pm
View file @ 160244e9
......@@ -188,16 +188,16 @@ sub avg_output_msec_per_job {
## other storable attributes:
sub last_update { # this method is called by the initial store() [at which point it returns undef]
sub when_updated { # this method is called by the initial store() [at which point it returns undef]
my $self = shift;
$self->{'_last_update'} = shift if(@_);
return $self->{'_last_update'};
$self->{'_when_updated'} = shift if(@_);
return $self->{'_when_updated'};
}
sub seconds_since_last_update { # this method is mostly used to convert between server time and local time
sub seconds_since_when_updated { # this method is mostly used to convert between server time and local time
my( $self, $value ) = @_;
$self->{'_last_update'} = time() - $value if(defined($value));
return defined($self->{'_last_update'}) ? time() - $self->{'_last_update'} : undef;
$self->{'_when_updated'} = time() - $value if(defined($value));
return defined($self->{'_when_updated'}) ? time() - $self->{'_when_updated'} : undef;
}
sub sync_lock {
......@@ -293,7 +293,7 @@ sub toString {
);
$output .= ' h.cap:' .( $self->hive_capacity // '-' )
.' a.cap:' .( $analysis->analysis_capacity // '-')
." (sync'd " .($self->seconds_since_last_update // 0)." sec ago)";
." (sync'd " .($self->seconds_since_when_updated // 0)." sec ago)";
return $output;
}
......
modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisJobAdaptor.pm
View file @ 160244e9
......@@ -289,11 +289,11 @@ sub check_in_job {
my $sql = "UPDATE job SET status='".$job->status."' ";
if($job->status eq 'DONE') {
$sql .= ",completed=CURRENT_TIMESTAMP";
$sql .= ",when_completed=CURRENT_TIMESTAMP";
$sql .= ",runtime_msec=".$job->runtime_msec;
$sql .= ",query_count=".$job->query_count;
} elsif($job->status eq 'PASSED_ON') {
$sql .= ", completed=CURRENT_TIMESTAMP";
$sql .= ", when_completed=CURRENT_TIMESTAMP";
} elsif($job->status eq 'READY') {
}
......
modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisStatsAdaptor.pm
View file @ 160244e9
......@@ -58,10 +58,10 @@ sub default_input_column_mapping {
my $self = shift @_;
my $driver = $self->dbc->driver();
return {
'last_update' => {
'mysql' => "UNIX_TIMESTAMP()-UNIX_TIMESTAMP(last_update) seconds_since_last_update ",
'sqlite' => "strftime('%s','now')-strftime('%s',last_update) seconds_since_last_update ",
'pgsql' => "EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - last_update) seconds_since_last_update ",
'when_updated' => {
'mysql' => "UNIX_TIMESTAMP()-UNIX_TIMESTAMP(when_updated) seconds_since_when_updated ",
'sqlite' => "strftime('%s','now')-strftime('%s',when_updated) seconds_since_when_updated ",
'pgsql' => "EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - when_updated) seconds_since_when_updated ",
}->{$driver},
};
}
......@@ -151,7 +151,7 @@ sub update {
}
$sql .= ",num_required_workers=" . $stats->num_required_workers();
$sql .= ",last_update=CURRENT_TIMESTAMP";
$sql .= ",when_updated=CURRENT_TIMESTAMP";
$sql .= ",sync_lock='0'";
$sql .= " WHERE analysis_id='".$stats->analysis_id."' ";
......@@ -161,7 +161,7 @@ sub update {
$sth = $self->prepare("INSERT INTO analysis_stats_monitor SELECT CURRENT_TIMESTAMP, analysis_stats.* from analysis_stats WHERE analysis_id = ".$stats->analysis_id);
$sth->execute();
$sth->finish;
$stats->seconds_since_last_update(0); #not exact but good enough :)
$stats->seconds_since_when_updated(0); #not exact but good enough :)
}
......
modules/Bio/EnsEMBL/Hive/DBSQL/LogMessageAdaptor.pm
View file @ 160244e9
......@@ -54,7 +54,7 @@ sub store_job_message {
my $table_name = $self->table_name();
# Note: the timestamp 'time' column will be set automatically
# Note: the timestamp 'when_logged' column will be set automatically
my $sql = qq{
INSERT INTO $table_name (job_id, role_id, worker_id, retry, status, msg, is_error)
SELECT job_id, role_id, worker_id, retry_count, status, ?, ?
......@@ -79,7 +79,7 @@ sub store_worker_message {
my $table_name = $self->table_name();
# Note: the timestamp 'time' column will be set automatically
# Note: the timestamp 'when_logged' column will be set automatically
my $sql = qq{
INSERT INTO $table_name (worker_id, role_id, status, msg, is_error)
SELECT worker_id, ?, status, ?, ?
......@@ -98,7 +98,7 @@ sub store_hive_message {
my $table_name = $self->table_name();
# Note: the timestamp 'time' column will be set automatically
# Note: the timestamp 'when_logged' column will be set automatically
my $sql = qq{
INSERT INTO $table_name (status, msg, is_error) VALUES ('UNKNOWN', ?, ?)
};
......
modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm
View file @ 160244e9
......@@ -219,10 +219,10 @@ sub parse_report_source_line {
if( my ($process_id) = $lines[0]=~/^Job <(\d+(?:\[\d+\])?)>/) {
my ($exit_status, $exception_status) = ('' x 2);
my ($died, $cause_of_death);
my ($when_died, $cause_of_death);
foreach (@lines) {
if( /^(\w+\s+\w+\s+\d+\s+\d+:\d+:\d+):\s+Completed\s<(\w+)>(?:\.|;\s+(\w+))/ ) {
$died = _yearless_2_datetime($1);
$when_died = _yearless_2_datetime($1);
$cause_of_death = $3 && $status_2_cod{$3};
$exit_status = $2 . ($3 ? "/$3" : '');
}
......@@ -243,7 +243,7 @@ sub parse_report_source_line {
$report_entry{ $process_id } = {
# entries for 'worker' table:
'died' => $died,
'when_died' => $when_died,
'cause_of_death' => $cause_of_death,
# entries for 'worker_resource_usage' table:
......
modules/Bio/EnsEMBL/Hive/Queen.pm
View file @ 160244e9
......@@ -287,12 +287,12 @@ sub specialize_worker {
sub register_worker_death {
my ($self, $worker, $update_last_check_in) = @_;
my ($self, $worker, $update_when_checked_in) = @_;
my $worker_id = $worker->dbID;
my $work_done = $worker->work_done;
my $cause_of_death = $worker->cause_of_death || 'UNKNOWN'; # make sure we do not attempt to insert a void
my $worker_died = $worker->died;
my $worker_died = $worker->when_died;
my $current_role = $worker->current_role;
......@@ -311,8 +311,8 @@ sub register_worker_death {
}
my $sql = "UPDATE worker SET status='DEAD', work_done='$work_done', cause_of_death='$cause_of_death'"
. ( $update_last_check_in ? ', last_check_in=CURRENT_TIMESTAMP ' : '' )
. ( $worker_died ? ", died='$worker_died'" : ', died=CURRENT_TIMESTAMP' )
. ( $update_when_checked_in ? ', when_checked_in=CURRENT_TIMESTAMP ' : '' )
. ( $worker_died ? ", when_died='$worker_died'" : ', when_died=CURRENT_TIMESTAMP' )
. " WHERE worker_id='$worker_id' ";
$self->dbc->do( $sql );
......@@ -386,7 +386,7 @@ sub check_for_dead_workers { # scans the whole Valley for lost Workers (but i
warn "GarbageCollector:\tReleasing the jobs\n";
while(my ($process_id, $worker) = each %$pid_to_lost_worker) {
$worker->died( $report_entries->{$process_id}{'died'} );
$worker->when_died( $report_entries->{$process_id}{'when_died'} );
$worker->cause_of_death( $report_entries->{$process_id}{'cause_of_death'} );
$self->register_worker_death( $worker );
}
......@@ -421,7 +421,7 @@ sub check_for_dead_workers { # scans the whole Valley for lost Workers (but i
sub check_in_worker {
my ($self, $worker) = @_;
$self->dbc->do("UPDATE worker SET last_check_in=CURRENT_TIMESTAMP, status='".$worker->status."', work_done='".$worker->work_done."' WHERE worker_id='".$worker->dbID."'");
$self->dbc->do("UPDATE worker SET when_checked_in=CURRENT_TIMESTAMP, status='".$worker->status."', work_done='".$worker->work_done."' WHERE worker_id='".$worker->dbID."'");
}
......@@ -469,9 +469,9 @@ sub fetch_overdue_workers {
$overdue_secs = 3600 unless(defined($overdue_secs));
my $constraint = "status!='DEAD' AND ".{
'mysql' => "(UNIX_TIMESTAMP()-UNIX_TIMESTAMP(last_check_in)) > $overdue_secs",
'sqlite' => "(strftime('%s','now')-strftime('%s',last_check_in)) > $overdue_secs",
'pgsql' => "EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - last_check_in) > $overdue_secs",
'mysql' => "(UNIX_TIMESTAMP()-UNIX_TIMESTAMP(when_checked_in)) > $overdue_secs",
'sqlite' => "(strftime('%s','now')-strftime('%s',when_checked_in)) > $overdue_secs",
'pgsql' => "EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - when_checked_in) > $overdue_secs",
}->{ $self->dbc->driver };
return $self->fetch_all( $constraint );
......@@ -531,7 +531,7 @@ sub safe_synchronize_AnalysisStats {
}
unless( ($stats->status eq 'DONE')
or ( ($stats->status eq 'WORKING') and defined($stats->seconds_since_last_update) and ($stats->seconds_since_last_update < 3*60) ) ) {
or ( ($stats->status eq 'WORKING') and defined($stats->seconds_since_when_updated) and ($stats->seconds_since_when_updated < 3*60) ) ) {
my $sql = "UPDATE analysis_stats SET status='SYNCHING', sync_lock=1 ".
"WHERE sync_lock=0 and analysis_id=" . $stats->analysis_id;
......@@ -681,7 +681,7 @@ sub interval_workers_with_unknown_usage {
my %meadow_to_interval = ();
my $sql_times = qq{
SELECT meadow_type, meadow_name, min(born), max(died), count(*)
SELECT meadow_type, meadow_name, min(when_born), max(when_died), count(*)
FROM worker w
LEFT JOIN worker_resource_usage u USING(worker_id)
WHERE u.worker_id IS NULL
......
modules/Bio/EnsEMBL/Hive/Worker.pm
View file @ 160244e9
......@@ -161,24 +161,31 @@ sub status {
}
sub born {
sub when_born {
my $self = shift;
$self->{'_born'} = shift if(@_);
return $self->{'_born'};
$self->{'_when_born'} = shift if(@_);
return $self->{'_when_born'};
}
sub last_check_in {
sub when_checked_in {
my $self = shift;
$self->{'_last_check_in'} = shift if(@_);
return $self->{'_last_check_in'};
$self->{'_when_checked_in'} = shift if(@_);
return $self->{'_when_checked_in'};
}
sub died {
sub when_seen {
my $self = shift;
$self->{'_died'} = shift if(@_);
return $self->{'_died'};
$self->{'_when_seen'} = shift if(@_);
return $self->{'_when_seen'};
}
sub when_died {
my $self = shift;
$self->{'_when_died'} = shift if(@_);
return $self->{'_when_died'};
}
......@@ -414,7 +421,7 @@ sub toString {
'resource_class_id='.($self->resource_class_id // 'NULL'),
'meadow='.$self->meadow_type.'/'.$self->meadow_name,
'process='.$self->meadow_user.'@'.$self->meadow_host.'#'.$self->process_id,
'last_check_in='.($self->last_check_in // 'NEVER'),
'when_checked_in='.($self->when_checked_in // 'NEVER'),
'batch_size='.($current_role ? $current_role->analysis->stats->get_or_estimate_batch_size() : 'UNSPECIALIZED'),
'job_limit='.($self->job_limiter->available_capacity() // 'NONE'),
'life_span='.($self->life_span // 'UNLIM'),
......@@ -561,8 +568,8 @@ sub run {
}
}
# The second argument ("update_last_check_in") is set to force an
# update of the "last_check_in" date in the worker table
# The second argument ("update_when_checked_in") is set to force an
# update of the "when_checked_in" timestamp in the worker table
$self->adaptor->register_worker_death($self, 1);
if($self->debug) {
......
scripts/generate_timeline.pl
View file @ 160244e9
......@@ -163,13 +163,13 @@ sub main {
my %layers = ();
{
my $sql = $key eq 'analysis'
? 'SELECT born, died, worker_id, resource_class_id, analysis_id FROM worker LEFT JOIN role USING (worker_id)'
: 'SELECT born, died, worker_id, resource_class_id FROM worker';
? 'SELECT when_born, when_died, worker_id, resource_class_id, analysis_id FROM worker LEFT JOIN role USING (worker_id)'
: 'SELECT when_born, when_died, worker_id, resource_class_id FROM worker';
my @tmp_dates = @{$dbh->selectall_arrayref($sql)};
warn scalar(@tmp_dates), " rows\n" if $verbose;
foreach my $db_entry (@tmp_dates) {
my ($born, $died, $worker_id, $resource_class_id, $analysis_id) = @$db_entry;
my ($when_born, $when_died, $worker_id, $resource_class_id, $analysis_id) = @$db_entry;
# In case $resource_class_id is undef
next unless $resource_class_id or $analysis_id;
......@@ -178,24 +178,24 @@ sub main {
$key_value = -1 if not defined $key_value;
if ($mode eq 'workers') {
add_event(\%events, $key_value, $born, $died, 1);
add_event(\%events, $key_value, $when_born, $when_died, 1);
} elsif ($mode eq 'memory') {
my $offset = ($mem_resources{$resource_class_id} || $default_memory) / 1024.;
add_event(\%events, $key_value, $born, $died, $offset);
add_event(\%events, $key_value, $when_born, $when_died, $offset);
$offset = ($used_res{$worker_id}->[0]) / 1024. if exists $used_res{$worker_id} and $used_res{$worker_id}->[0];
add_event(\%layers, $key_value, $born, $died, $offset);
add_event(\%layers, $key_value, $when_born, $when_died, $offset);
} elsif ($mode eq 'cores') {
my $offset = ($cpu_resources{$resource_class_id} || $default_cores);
add_event(\%events, $key_value, $born, $died, $offset);
add_event(\%events, $key_value, $when_born, $when_died, $offset);
$offset = $used_res{$worker_id}->[1] if exists $used_res{$worker_id} and $used_res{$worker_id}->[1];
add_event(\%layers, $key_value, $born, $died, $offset);
add_event(\%layers, $key_value, $when_born, $when_died, $offset);
} else {
if (exists $used_res{$worker_id} and $used_res{$worker_id}->[2]) {
my $pending_sec = $used_res{$worker_id}->[2];
add_event(\%events, $key_value, -$pending_sec, $born, 1);
add_event(\%layers, $key_value, -$pending_sec, $born, $pending_sec/60);
add_event(\%events, $key_value, -$pending_sec, $when_born, 1);
add_event(\%layers, $key_value, -$pending_sec, $when_born, $pending_sec/60);
}
}
}
......@@ -349,13 +349,13 @@ sub add_dataset {
#####
sub add_event {
my ($events, $key, $born, $died, $offset) = @_;
my ($events, $key, $when_born, $when_died, $offset) = @_;
return if $offset <= 0;
# temporary Time::Piece values
my $death_datetime = Time::Piece->strptime( $died , '%Y-%m-%d %H:%M:%S');
my $birth_datetime = ($born =~ /^-[0-9]/) ? $death_datetime + $born : Time::Piece->strptime( $born , '%Y-%m-%d %H:%M:%S');
my $death_datetime = Time::Piece->strptime( $when_died , '%Y-%m-%d %H:%M:%S');
my $birth_datetime = ($when_born =~ /^-[0-9]/) ? $death_datetime + $when_born : Time::Piece->strptime( $when_born , '%Y-%m-%d %H:%M:%S');
# We don't need to draw things at the resolution of 1 second; 1 minute is enough
$death_datetime->[0] = 0;
......@@ -364,10 +364,10 @@ sub add_event {
# string values:
my $birth_date = $birth_datetime->date . 'T' . $birth_datetime->hms;
my $death_date = $death_datetime->date . 'T' . $death_datetime->hms;
return if $died and ($birth_date eq $death_date);
return if $when_died and ($birth_date eq $death_date);
$events->{$birth_date}{$key} += $offset;
$events->{$death_date}{$key} -= $offset if $died;
$events->{$death_date}{$key} -= $offset if $when_died;
}
......
scripts/hoover_pipeline.pl
View file @ 160244e9
......@@ -63,7 +63,7 @@ sub main {
my $sql = qq{
DELETE j FROM job j
WHERE j.status='DONE'
AND j.completed < $threshold_datetime_expression
AND j.when_completed < $threshold_datetime_expression
};
my $dbc = $hive_dba->dbc();
......
sql/patch_2015-01-27.mysql 0 → 100644
View file @ 160244e9
-- First, rename all TIMESTAMPed columns to have a 'when_' prefix for automatic identification:
ALTER TABLE analysis_stats CHANGE COLUMN last_update when_updated TIMESTAMP NULL;
ALTER TABLE job CHANGE COLUMN completed when_completed TIMESTAMP NULL;
ALTER TABLE worker CHANGE COLUMN born when_born TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
ALTER TABLE worker CHANGE COLUMN last_check_in when_checked_in TIMESTAMP NULL;
ALTER TABLE worker CHANGE COLUMN died when_died TIMESTAMP NULL;
ALTER TABLE log_message CHANGE COLUMN time when_logged TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
ALTER TABLE analysis_stats_monitor CHANGE COLUMN time when_logged TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
ALTER TABLE analysis_stats_monitor CHANGE COLUMN last_update when_updated TIMESTAMP NULL;
-- Then add one more column to register when a Worker was last seen by the Meadow:
ALTER TABLE worker ADD COLUMN when_seen TIMESTAMP NULL AFTER when_checked_in;
-- replace the 'msg' view as the columns implicitly referenced there have been renamed:
CREATE OR REPLACE VIEW msg AS
SELECT a.analysis_id, a.logic_name, m.*
FROM log_message m
LEFT JOIN job j ON (j.job_id=m.job_id)
LEFT JOIN analysis_base a ON (a.analysis_id=j.analysis_id);
-- UPDATE hive_sql_schema_version
UPDATE hive_meta SET meta_value=64 WHERE meta_key='hive_sql_schema_version' AND meta_value='63';
sql/patch_2015-01-27.pgsql 0 → 100644
View file @ 160244e9
-- First, rename all TIMESTAMPed columns to have a 'when_' prefix for automatic identification:
ALTER TABLE analysis_stats RENAME COLUMN last_update TO when_updated;
ALTER TABLE job RENAME COLUMN completed TO when_completed;
ALTER TABLE worker RENAME COLUMN born TO when_born;
ALTER TABLE worker RENAME COLUMN last_check_in TO when_checked_in;
ALTER TABLE worker RENAME COLUMN died TO when_died;
ALTER TABLE log_message RENAME COLUMN time TO when_logged;
ALTER TABLE analysis_stats_monitor RENAME COLUMN time TO when_logged;
ALTER TABLE analysis_stats_monitor RENAME COLUMN last_update TO when_updated;
-- Then add one more column to register when a Worker was last seen by the Meadow:
ALTER TABLE worker ADD COLUMN when_seen TIMESTAMP DEFAULT NULL;
-- replace the 'msg' view as the columns implicitly referenced there have been renamed:
DROP VIEW msg;
CREATE OR REPLACE VIEW msg AS
SELECT a.analysis_id, a.logic_name, m.*
FROM log_message m
LEFT JOIN job j ON (j.job_id=m.job_id)
LEFT JOIN analysis_base a ON (a.analysis_id=j.analysis_id);
-- UPDATE hive_sql_schema_version
UPDATE hive_meta SET meta_value=64 WHERE meta_key='hive_sql_schema_version' AND meta_value='63';
sql/tables.mysql
View file @ 160244e9
......@@ -148,7 +148,7 @@ CREATE TABLE analysis_base (
@column avg_run_msec_per_job weighted average used to compute DYNAMIC hive_capacity
@column avg_output_msec_per_job weighted average used to compute DYNAMIC hive_capacity
@column last_update when this entry was last updated
@column when_updated when this entry was last updated
@column sync_lock a binary lock flag to prevent simultaneous updates
*/
......@@ -175,7 +175,7 @@ CREATE TABLE analysis_stats (
avg_run_msec_per_job INTEGER DEFAULT NULL,
avg_output_msec_per_job INTEGER DEFAULT NULL,
last_update TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
when_updated TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
sync_lock SMALLINT NOT NULL DEFAULT 0,
PRIMARY KEY (analysis_id)
......@@ -315,7 +315,7 @@ CREATE TABLE resource_description (
@column role_id links to the Role that claimed this job (NULL means it has never been claimed)
@column status state the job is in
@column retry_count number times job had to be reset when worker failed to run it
@column completed when the job was completed
@column when_completed when the job was completed
@column runtime_msec how long did it take to execute the job (or until the moment it failed)
@column query_count how many SQL queries were run during this job
@column semaphore_count if this count is >0, the job is conditionally blocked (until this count drops to 0 or below). Default=0 means "nothing is blocking me by default".
......@@ -332,7 +332,7 @@ CREATE TABLE job (
role_id INTEGER DEFAULT NULL,
status ENUM('SEMAPHORED','READY','CLAIMED','COMPILATION','PRE_CLEANUP','FETCH_INPUT','RUN','WRITE_OUTPUT','POST_CLEANUP','DONE','FAILED','PASSED_ON') DEFAULT 'READY' NOT NULL,
retry_count INTEGER NOT NULL DEFAULT 0,
completed TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
when_completed TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
runtime_msec INTEGER DEFAULT NULL,
query_count INTEGER DEFAULT NULL,
......@@ -446,9 +446,10 @@ CREATE TABLE analysis_data (
@column resource_class_id links to Worker's resource class
@column work_done how many jobs the Worker has completed successfully
@column status current status of the Worker
@column born when the Worker process was started
@column last_check_in when the Worker last checked into the database
@column died if defined, when the Worker died (or its premature death was first detected by GC)
@column when_born when the Worker process was started
@column when_checked_in when the Worker last checked into the database
@column when_seen when the Worker was last seen by the Meadow
@column when_died if defined, when the Worker died (or its premature death was first detected by GC)
@column cause_of_death if defined, why did the Worker exit (or why it was killed)
@column log_dir if defined, a filesystem directory where this Worker's output is logged
*/
......@@ -464,9 +465,10 @@ CREATE TABLE worker (
work_done INTEGER NOT NULL DEFAULT 0,
status ENUM('SPECIALIZATION','COMPILATION','READY','PRE_CLEANUP','FETCH_INPUT','RUN','WRITE_OUTPUT','POST_CLEANUP','DEAD') DEFAULT 'READY' NOT NULL,
born TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_check_in TIMESTAMP NULL,
died TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
when_born TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_checked_in TIMESTAMP NULL,
when_seen TIMESTAMP NULL,
when_died TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
cause_of_death ENUM('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN') DEFAULT NULL,
log_dir VARCHAR(255) DEFAULT NULL,
......@@ -556,7 +558,7 @@ CREATE TABLE worker_resource_usage (
@column job_id the id of the job that threw the message (or NULL if it was outside of a message)
@column role_id the 'current' role
@column worker_id the 'current' worker
@column time when the message was thrown
@column when_logged when the message was thrown
@column retry retry_count of the job when the message was thrown (or NULL if no job)
@column status of the job or worker when the message was thrown
@column msg string that contains the message
......@@ -568,7 +570,7 @@ CREATE TABLE log_message (
job_id INTEGER DEFAULT NULL,
role_id INTEGER DEFAULT NULL,
worker_id INTEGER DEFAULT NULL,
time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_logged TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
retry INTEGER DEFAULT NULL,
status ENUM('UNKNOWN','SPECIALIZATION','COMPILATION','CLAIMED','READY','PRE_CLEANUP','FETCH_INPUT','RUN','WRITE_OUTPUT','POST_CLEANUP','PASSED_ON') DEFAULT 'UNKNOWN',
msg TEXT,
......@@ -586,7 +588,7 @@ CREATE TABLE log_message (
@desc A regular timestamped snapshot of the analysis_stats table.
@column time when this snapshot was taken
@column when_logged when this snapshot was taken
@column analysis_id foreign-keyed to the corresponding analysis_base entry
@column batch_size how many jobs are claimed in one claiming operation before Worker starts executing them
......@@ -611,12 +613,12 @@ CREATE TABLE log_message (
@column avg_run_msec_per_job weighted average used to compute DYNAMIC hive_capacity
@column avg_output_msec_per_job weighted average used to compute DYNAMIC hive_capacity
@column last_update when this entry was last updated
@column when_updated when this entry was last updated
@column sync_lock a binary lock flag to prevent simultaneous updates
*/
CREATE TABLE analysis_stats_monitor (
time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_logged TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
analysis_id INTEGER NOT NULL,
batch_size INTEGER NOT NULL DEFAULT 1,
......@@ -640,7 +642,7 @@ CREATE TABLE analysis_stats_monitor (
avg_run_msec_per_job INTEGER DEFAULT NULL,
avg_output_msec_per_job INTEGER DEFAULT NULL,
last_update TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
when_updated TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
sync_lock SMALLINT NOT NULL DEFAULT 0
) COLLATE=latin1_swedish_ci ENGINE=InnoDB;
......
sql/tables.pgsql
View file @ 160244e9
......@@ -150,7 +150,7 @@ CREATE TABLE analysis_base (
@column avg_run_msec_per_job weighted average used to compute DYNAMIC hive_capacity
@column avg_output_msec_per_job weighted average used to compute DYNAMIC hive_capacity
@column last_update when this entry was last updated
@column when_updated when this entry was last updated
@column sync_lock a binary lock flag to prevent simultaneous updates
*/
......@@ -179,7 +179,7 @@ CREATE TABLE analysis_stats (
avg_run_msec_per_job INTEGER DEFAULT NULL,
avg_output_msec_per_job INTEGER DEFAULT NULL,
last_update TIMESTAMP DEFAULT NULL,
when_updated TIMESTAMP DEFAULT NULL,
sync_lock SMALLINT NOT NULL DEFAULT 0,
PRIMARY KEY (analysis_id)
......@@ -319,7 +319,7 @@ CREATE TABLE resource_description (
@column role_id links to the Role that claimed this job (NULL means it has never been claimed)
@column status state the job is in
@column retry_count number times job had to be reset when worker failed to run it
@column completed when the job was completed
@column when_completed when the job was completed
@column runtime_msec how long did it take to execute the job (or until the moment it failed)
@column query_count how many SQL queries were run during this job
@column semaphore_count if this count is >0, the job is conditionally blocked (until this count drops to 0 or below). Default=0 means "nothing is blocking me by default".
......@@ -339,7 +339,7 @@ CREATE TABLE job (
role_id INTEGER DEFAULT NULL,
status jw_status NOT NULL DEFAULT 'READY',
retry_count INTEGER NOT NULL DEFAULT 0,
completed TIMESTAMP DEFAULT NULL,
when_completed TIMESTAMP DEFAULT NULL,
runtime_msec INTEGER DEFAULT NULL,
query_count INTEGER DEFAULT NULL,
......@@ -451,9 +451,10 @@ CREATE INDEX ON analysis_data (data);
@column resource_class_id links to Worker's resource class
@column work_done how many jobs the Worker has completed successfully
@column status current status of the Worker
@column born when the Worker process was started
@column last_check_in when the Worker last checked into the database
@column died if defined, when the Worker died (or its premature death was first detected by GC)
@column when_born when the Worker process was started
@column when_checked_in when the Worker last checked into the database
@column when_seen when the Worker was last seen by the Meadow
@column when_died if defined, when the Worker died (or its premature death was first detected by GC)
@column cause_of_death if defined, why did the Worker exit (or why it was killed)
@column log_dir if defined, a filesystem directory where this Worker's output is logged
*/
......@@ -469,9 +470,10 @@ CREATE TABLE worker (
resource_class_id INTEGER DEFAULT NULL,
work_done INTEGER NOT NULL DEFAULT 0,
status jw_status NOT NULL DEFAULT 'READY',
born TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_check_in TIMESTAMP DEFAULT NULL,
died TIMESTAMP DEFAULT NULL,
when_born TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_checked_in TIMESTAMP DEFAULT NULL,
when_seen TIMESTAMP DEFAULT NULL,
when_died TIMESTAMP DEFAULT NULL,
cause_of_death worker_cod DEFAULT NULL,
log_dir VARCHAR(255) DEFAULT NULL
);
......@@ -560,7 +562,7 @@ CREATE TABLE worker_resource_usage (
@column job_id the id of the job that threw the message (or NULL if it was outside of a message)
@column role_id the 'current' role
@column worker_id the 'current' worker
@column time when the message was thrown
@column when_logged when the message was thrown
@column retry retry_count of the job when the message was thrown (or NULL if no job)
@column status of the job or worker when the message was thrown
@column msg string that contains the message
......@@ -572,7 +574,7 @@ CREATE TABLE log_message (
job_id INTEGER DEFAULT NULL,
role_id INTEGER DEFAULT NULL,
worker_id INTEGER DEFAULT NULL,
time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_logged TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
retry INTEGER DEFAULT NULL,
status jw_status DEFAULT 'UNKNOWN',
msg TEXT,
......@@ -590,7 +592,7 @@ CREATE INDEX ON log_message (job_id);
@desc A regular timestamped snapshot of the analysis_stats table.