Skip to content

GitLab

Commit 8739d4ee authored by Leo Gordon's avatar Leo Gordon
Browse files

added specific defaults into analysis_stats and analysis_stats_monitor;... 

added specific defaults into analysis_stats and analysis_stats_monitor; re-ordered the fields for easier navigation
parent aa224e93
Hide whitespace changes
Inline Side-by-side
Showing with 200 additions and 164 deletions
modules/Bio/EnsEMBL/Hive/AnalysisStats.pm
View file @ 8739d4ee
......@@ -127,37 +127,50 @@ sub status {
}
## non-presettable storable object's getters/setters:
## counters of jobs in different states:
sub avg_msec_per_job {
sub total_job_count {
my $self = shift;
$self->{'_avg_msec_per_job'} = shift if(@_);
$self->{'_avg_msec_per_job'}=0 unless($self->{'_avg_msec_per_job'});
return $self->{'_avg_msec_per_job'};
$self->{'_total_job_count'} = shift if(@_);
return $self->{'_total_job_count'};
}
sub avg_input_msec_per_job {
sub unclaimed_job_count {
my $self = shift;
$self->{'_avg_input_msec_per_job'} = shift if(@_);
$self->{'_avg_input_msec_per_job'}=0 unless($self->{'_avg_input_msec_per_job'});
return $self->{'_avg_input_msec_per_job'};
$self->{'_unclaimed_job_count'} = shift if(@_);
return $self->{'_unclaimed_job_count'};
}
sub avg_run_msec_per_job {
sub done_job_count {
my $self = shift;
$self->{'_avg_run_msec_per_job'} = shift if(@_);
$self->{'_avg_run_msec_per_job'}=0 unless($self->{'_avg_run_msec_per_job'});
return $self->{'_avg_run_msec_per_job'};
$self->{'_done_job_count'} = shift if(@_);
return $self->{'_done_job_count'};
}
sub avg_output_msec_per_job {
sub failed_job_count {
my $self = shift;
$self->{'_avg_output_msec_per_job'} = shift if(@_);
$self->{'_avg_output_msec_per_job'}=0 unless($self->{'_avg_output_msec_per_job'});
return $self->{'_avg_output_msec_per_job'};
$self->{'_failed_job_count'} = shift if(@_);
$self->{'_failed_job_count'} = 0 unless(defined($self->{'_failed_job_count'}));
return $self->{'_failed_job_count'};
}
sub num_running_workers {
my $self = shift;
$self->{'_num_running_workers'} = shift if(@_);
return $self->{'_num_running_workers'};
}
sub num_required_workers {
my $self = shift;
$self->{'_num_required_workers'} = shift if(@_);
return $self->{'_num_required_workers'};
}
## dynamic hive_capacity mode attributes:
sub behaviour {
my $self = shift;
$self->{'_behaviour'} = shift if(@_);
......@@ -176,43 +189,42 @@ sub output_capacity {
return $self->{'_output_capacity'};
}
sub total_job_count {
my $self = shift;
$self->{'_total_job_count'} = shift if(@_);
return $self->{'_total_job_count'};
}
sub unclaimed_job_count {
my $self = shift;
$self->{'_unclaimed_job_count'} = shift if(@_);
return $self->{'_unclaimed_job_count'};
}
## dynamic hive_capacity mode counters:
sub done_job_count {
sub avg_msec_per_job {
my $self = shift;
$self->{'_done_job_count'} = shift if(@_);
return $self->{'_done_job_count'};
$self->{'_avg_msec_per_job'} = shift if(@_);
$self->{'_avg_msec_per_job'}=0 unless($self->{'_avg_msec_per_job'});
return $self->{'_avg_msec_per_job'};
}
sub failed_job_count {
sub avg_input_msec_per_job {
my $self = shift;
$self->{'_failed_job_count'} = shift if(@_);
$self->{'_failed_job_count'} = 0 unless(defined($self->{'_failed_job_count'}));
return $self->{'_failed_job_count'};
$self->{'_avg_input_msec_per_job'} = shift if(@_);
$self->{'_avg_input_msec_per_job'}=0 unless($self->{'_avg_input_msec_per_job'});
return $self->{'_avg_input_msec_per_job'};
}
sub num_running_workers {
sub avg_run_msec_per_job {
my $self = shift;
$self->{'_num_running_workers'} = shift if(@_);
return $self->{'_num_running_workers'};
$self->{'_avg_run_msec_per_job'} = shift if(@_);
$self->{'_avg_run_msec_per_job'}=0 unless($self->{'_avg_run_msec_per_job'});
return $self->{'_avg_run_msec_per_job'};
}
sub num_required_workers {
sub avg_output_msec_per_job {
my $self = shift;
$self->{'_num_required_workers'} = shift if(@_);
return $self->{'_num_required_workers'};
$self->{'_avg_output_msec_per_job'} = shift if(@_);
$self->{'_avg_output_msec_per_job'}=0 unless($self->{'_avg_output_msec_per_job'});
return $self->{'_avg_output_msec_per_job'};
}
## other storable ttributes:
sub seconds_since_last_update {
my( $self, $value ) = @_;
$self->{'_last_update'} = time() - $value if(defined($value));
......
modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisStatsAdaptor.pm
View file @ 8739d4ee
......@@ -409,28 +409,32 @@ sub _columns {
my $self = shift;
my @columns = qw (ast.analysis_id
ast.status
ast.batch_size
ast.avg_msec_per_job
ast.avg_input_msec_per_job
ast.avg_run_msec_per_job
ast.avg_output_msec_per_job
ast.hive_capacity
ast.behaviour
ast.input_capacity
ast.output_capacity
ast.failed_job_tolerance
ast.max_retry_count
ast.can_be_empty
ast.priority
ast.status
ast.total_job_count
ast.unclaimed_job_count
ast.done_job_count
ast.max_retry_count
ast.failed_job_count
ast.failed_job_tolerance
ast.num_running_workers
ast.num_required_workers
ast.behaviour
ast.input_capacity
ast.output_capacity
ast.avg_msec_per_job
ast.avg_input_msec_per_job
ast.avg_run_msec_per_job
ast.avg_output_msec_per_job
ast.last_update
ast.sync_lock
ast.can_be_empty
ast.priority
);
push @columns , ($self->dbc->driver eq 'sqlite')
......@@ -451,28 +455,32 @@ sub _objs_from_sth {
my $analStats = Bio::EnsEMBL::Hive::AnalysisStats->new();
$analStats->analysis_id($column{'analysis_id'});
$analStats->status($column{'status'});
$analStats->sync_lock($column{'sync_lock'});
$analStats->can_be_empty($column{'can_be_empty'});
$analStats->priority($column{'priority'});
$analStats->batch_size($column{'batch_size'});
$analStats->avg_msec_per_job($column{'avg_msec_per_job'});
$analStats->avg_input_msec_per_job($column{'avg_input_msec_per_job'});
$analStats->avg_run_msec_per_job($column{'avg_run_msec_per_job'});
$analStats->avg_output_msec_per_job($column{'avg_output_msec_per_job'});
$analStats->hive_capacity($column{'hive_capacity'});
$analStats->behaviour($column{'behaviour'});
$analStats->input_capacity($column{'input_capacity'});
$analStats->output_capacity($column{'output_capacity'});
$analStats->failed_job_tolerance($column{'failed_job_tolerance'});
$analStats->max_retry_count($column{'max_retry_count'});
$analStats->can_be_empty($column{'can_be_empty'});
$analStats->priority($column{'priority'});
$analStats->status($column{'status'});
$analStats->total_job_count($column{'total_job_count'});
$analStats->unclaimed_job_count($column{'unclaimed_job_count'});
$analStats->done_job_count($column{'done_job_count'});
$analStats->max_retry_count($column{'max_retry_count'});
$analStats->failed_job_count($column{'failed_job_count'});
$analStats->failed_job_tolerance($column{'failed_job_tolerance'});
$analStats->num_running_workers($column{'num_running_workers'});
$analStats->num_required_workers($column{'num_required_workers'});
$analStats->behaviour($column{'behaviour'});
$analStats->input_capacity($column{'input_capacity'});
$analStats->output_capacity($column{'output_capacity'});
$analStats->avg_msec_per_job($column{'avg_msec_per_job'});
$analStats->avg_input_msec_per_job($column{'avg_input_msec_per_job'});
$analStats->avg_run_msec_per_job($column{'avg_run_msec_per_job'});
$analStats->avg_output_msec_per_job($column{'avg_output_msec_per_job'});
$analStats->seconds_since_last_update($column{'seconds_since_last_update'});
$analStats->sync_lock($column{'sync_lock'});
$analStats->adaptor($self);
......
sql/tables.sql
View file @ 8739d4ee
......@@ -323,68 +323,75 @@ CREATE TABLE resource_description (
-- analysis_id - foreign key to analysis table
-- status - overview status of the jobs (cached state)
-- failed_job_tolerance - % of tolerated failed jobs
-- resource_class_id - resource class id (analyses are grouped into disjoint classes)
CREATE TABLE analysis_stats (
analysis_id int(10) unsigned NOT NULL,
status enum('BLOCKED', 'LOADING', 'SYNCHING', 'READY', 'WORKING', 'ALL_CLAIMED', 'DONE', 'FAILED')
DEFAULT 'READY' NOT NULL,
batch_size int(10) default 1 NOT NULL,
avg_msec_per_job int(10) default 0 NOT NULL,
avg_input_msec_per_job int(10) default 0 NOT NULL,
avg_run_msec_per_job int(10) default 0 NOT NULL,
avg_output_msec_per_job int(10) default 0 NOT NULL,
hive_capacity int(10) default 1 NOT NULL,
behaviour enum('STATIC', 'DYNAMIC') DEFAULT 'STATIC' NOT NULL,
input_capacity int(10) default 4 NOT NULL,
output_capacity int(10) default 4 NOT NULL,
total_job_count int(10) NOT NULL,
unclaimed_job_count int(10) NOT NULL,
done_job_count int(10) NOT NULL,
max_retry_count int(10) default 3 NOT NULL,
failed_job_count int(10) NOT NULL,
failed_job_tolerance int(10) default 0 NOT NULL,
num_running_workers int(10) default 0 NOT NULL,
num_required_workers int(10) NOT NULL,
last_update datetime NOT NULL,
sync_lock int(10) default 0 NOT NULL,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
priority TINYINT DEFAULT 0 NOT NULL,
UNIQUE KEY (analysis_id)
analysis_id int(10) unsigned NOT NULL,
batch_size int(10) DEFAULT 1 NOT NULL,
hive_capacity int(10) DEFAULT 1 NOT NULL,
failed_job_tolerance int(10) DEFAULT 0 NOT NULL,
max_retry_count int(10) DEFAULT 3 NOT NULL,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
priority TINYINT DEFAULT 0 NOT NULL,
status enum('BLOCKED', 'LOADING', 'SYNCHING', 'READY', 'WORKING', 'ALL_CLAIMED', 'DONE', 'FAILED') DEFAULT 'READY' NOT NULL,
total_job_count int(10) DEFAULT NULL,
unclaimed_job_count int(10) DEFAULT NULL,
done_job_count int(10) DEFAULT NULL,
failed_job_count int(10) DEFAULT NULL,
num_running_workers int(10) DEFAULT NULL,
num_required_workers int(10) DEFAULT NULL,
behaviour enum('STATIC', 'DYNAMIC') DEFAULT 'STATIC' NOT NULL,
input_capacity int(10) DEFAULT 4 NOT NULL,
output_capacity int(10) DEFAULT 4 NOT NULL,
avg_msec_per_job int(10) DEFAULT NULL,
avg_input_msec_per_job int(10) DEFAULT NULL,
avg_run_msec_per_job int(10) DEFAULT NULL,
avg_output_msec_per_job int(10) DEFAULT NULL,
last_update datetime NOT NULL,
sync_lock int(10) default 0 NOT NULL,
UNIQUE KEY (analysis_id)
) COLLATE=latin1_swedish_ci ENGINE=InnoDB;
CREATE TABLE analysis_stats_monitor (
time datetime NOT NULL default '0000-00-00 00:00:00',
analysis_id int(10) unsigned NOT NULL,
status enum('BLOCKED', 'LOADING', 'SYNCHING', 'READY', 'WORKING', 'ALL_CLAIMED', 'DONE', 'FAILED')
DEFAULT 'READY' NOT NULL,
batch_size int(10) default 1 NOT NULL,
avg_msec_per_job int(10) default 0 NOT NULL,
avg_input_msec_per_job int(10) default 0 NOT NULL,
avg_run_msec_per_job int(10) default 0 NOT NULL,
avg_output_msec_per_job int(10) default 0 NOT NULL,
hive_capacity int(10) default 1 NOT NULL,
behaviour enum('STATIC', 'DYNAMIC') DEFAULT 'STATIC' NOT NULL,
input_capacity int(10) default 4 NOT NULL,
output_capacity int(10) default 4 NOT NULL,
total_job_count int(10) NOT NULL,
unclaimed_job_count int(10) NOT NULL,
done_job_count int(10) NOT NULL,
max_retry_count int(10) default 3 NOT NULL,
failed_job_count int(10) NOT NULL,
failed_job_tolerance int(10) default 0 NOT NULL,
num_running_workers int(10) default 0 NOT NULL,
num_required_workers int(10) NOT NULL,
last_update datetime NOT NULL,
sync_lock int(10) default 0 NOT NULL,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
priority TINYINT DEFAULT 0 NOT NULL
time datetime NOT NULL default '0000-00-00 00:00:00',
analysis_id int(10) unsigned NOT NULL,
batch_size int(10) DEFAULT 1 NOT NULL,
hive_capacity int(10) DEFAULT 1 NOT NULL,
failed_job_tolerance int(10) DEFAULT 0 NOT NULL,
max_retry_count int(10) DEFAULT 3 NOT NULL,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
priority TINYINT DEFAULT 0 NOT NULL,
status enum('BLOCKED', 'LOADING', 'SYNCHING', 'READY', 'WORKING', 'ALL_CLAIMED', 'DONE', 'FAILED') DEFAULT 'READY' NOT NULL,
total_job_count int(10) DEFAULT NULL,
unclaimed_job_count int(10) DEFAULT NULL,
done_job_count int(10) DEFAULT NULL,
failed_job_count int(10) DEFAULT NULL,
num_running_workers int(10) DEFAULT NULL,
num_required_workers int(10) DEFAULT NULL,
behaviour enum('STATIC', 'DYNAMIC') DEFAULT 'STATIC' NOT NULL,
input_capacity int(10) DEFAULT 4 NOT NULL,
output_capacity int(10) DEFAULT 4 NOT NULL,
avg_msec_per_job int(10) DEFAULT NULL,
avg_input_msec_per_job int(10) DEFAULT NULL,
avg_run_msec_per_job int(10) DEFAULT NULL,
avg_output_msec_per_job int(10) DEFAULT NULL,
last_update datetime NOT NULL,
sync_lock int(10) default 0 NOT NULL
) COLLATE=latin1_swedish_ci ENGINE=InnoDB;
-- ---------------------------------------------------------------------------------
--
-- Table structure for table 'monitor'
......
sql/tables.sqlite
View file @ 8739d4ee
......@@ -304,58 +304,67 @@ CREATE TABLE resource_description (
-- resource_class_id - resource class id (analyses are grouped into disjoint classes)
CREATE TABLE analysis_stats (
analysis_id INTEGER NOT NULL,
status TEXT DEFAULT 'READY' NOT NULL, /* enum('BLOCKED', 'LOADING', 'SYNCHING', 'READY', 'WORKING', 'ALL_CLAIMED', 'DONE', 'FAILED') DEFAULT 'READY' NOT NULL, */
batch_size int(10) default 1 NOT NULL,
avg_msec_per_job int(10) default 0 NOT NULL,
avg_input_msec_per_job int(10) default 0 NOT NULL,
avg_run_msec_per_job int(10) default 0 NOT NULL,
avg_output_msec_per_job int(10) default 0 NOT NULL,
hive_capacity int(10) default 1 NOT NULL,
behaviour TEXT DEFAULT 'STATIC' NOT NULL, /* enum('STATIC', 'DYNAMIC') DEFAULT 'STATIC' NOT NULL, */
input_capacity int(10) NOT NULL DEFAULT 4,
output_capacity int(10) NOT NULL DEFAULT 4,
total_job_count int(10) NOT NULL DEFAULT 0,
unclaimed_job_count int(10) NOT NULL DEFAULT 0,
done_job_count int(10) NOT NULL DEFAULT 0,
max_retry_count int(10) NOT NULL DEFAULT 3,
failed_job_count int(10) NOT NULL DEFAULT 0,
failed_job_tolerance int(10) NOT NULL DEFAULT 0,
num_running_workers int(10) NOT NULL DEFAULT 0,
num_required_workers int(10) NOT NULL DEFAULT 0,
last_update datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
sync_lock int(10) NOT NULL DEFAULT 0,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
priority TINYINT DEFAULT 0 NOT NULL
analysis_id INTEGER NOT NULL,
batch_size int(10) default 1 NOT NULL,
hive_capacity int(10) default 1 NOT NULL,
failed_job_tolerance int(10) NOT NULL DEFAULT 0,
max_retry_count int(10) NOT NULL DEFAULT 3,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
priority TINYINT DEFAULT 0 NOT NULL,
status TEXT DEFAULT 'READY' NOT NULL, /* enum('BLOCKED', 'LOADING', 'SYNCHING', 'READY', 'WORKING', 'ALL_CLAIMED', 'DONE', 'FAILED') DEFAULT 'READY' NOT NULL, */
total_job_count int(10) NOT NULL DEFAULT 0,
unclaimed_job_count int(10) NOT NULL DEFAULT 0,
done_job_count int(10) NOT NULL DEFAULT 0,
failed_job_count int(10) NOT NULL DEFAULT 0,
num_running_workers int(10) NOT NULL DEFAULT 0,
num_required_workers int(10) NOT NULL DEFAULT 0,
behaviour TEXT DEFAULT 'STATIC' NOT NULL, /* enum('STATIC', 'DYNAMIC') DEFAULT 'STATIC' NOT NULL, */
input_capacity int(10) NOT NULL DEFAULT 4,
output_capacity int(10) NOT NULL DEFAULT 4,
avg_msec_per_job int(10) default 0 NOT NULL,
avg_input_msec_per_job int(10) default 0 NOT NULL,
avg_run_msec_per_job int(10) default 0 NOT NULL,
avg_output_msec_per_job int(10) default 0 NOT NULL,
last_update datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
sync_lock int(10) NOT NULL DEFAULT 0
);
CREATE UNIQUE INDEX IF NOT EXISTS analysis_idx ON analysis_stats (analysis_id);
CREATE TABLE analysis_stats_monitor (
time datetime NOT NULL default '0000-00-00 00:00:00',
analysis_id INTEGER NOT NULL,
status TEXT DEFAULT 'READY' NOT NULL, /* enum('BLOCKED', 'LOADING', 'SYNCHING', 'READY', 'WORKING', 'ALL_CLAIMED', 'DONE', 'FAILED') DEFAULT 'READY' NOT NULL, */
batch_size int(10) default 1 NOT NULL,
avg_msec_per_job int(10) default 0 NOT NULL,
avg_input_msec_per_job int(10) default 0 NOT NULL,
avg_run_msec_per_job int(10) default 0 NOT NULL,
avg_output_msec_per_job int(10) default 0 NOT NULL,
hive_capacity int(10) default 1 NOT NULL,
behaviour TEXT DEFAULT 'STATIC' NOT NULL, /* enum('STATIC', 'DYNAMIC') DEFAULT 'STATIC' NOT NULL, */
input_capacity int(10) default 4 NOT NULL,
output_capacity int(10) default 4 NOT NULL,
total_job_count int(10) NOT NULL DEFAULT 0,
unclaimed_job_count int(10) NOT NULL DEFAULT 0,
done_job_count int(10) NOT NULL DEFAULT 0,
max_retry_count int(10) default 3 NOT NULL,
failed_job_count int(10) NOT NULL DEFAULT 0,
failed_job_tolerance int(10) default 0 NOT NULL,
num_running_workers int(10) default 0 NOT NULL,
num_required_workers int(10) NOT NULL DEFAULT 0,
last_update datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
sync_lock int(10) default 0 NOT NULL,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
priority TINYINT DEFAULT 0 NOT NULL
time datetime NOT NULL default '0000-00-00 00:00:00',
analysis_id INTEGER NOT NULL,
batch_size int(10) default 1 NOT NULL,
hive_capacity int(10) default 1 NOT NULL,
failed_job_tolerance int(10) default 0 NOT NULL,
max_retry_count int(10) default 3 NOT NULL,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
priority TINYINT DEFAULT 0 NOT NULL,
status TEXT DEFAULT 'READY' NOT NULL, /* enum('BLOCKED', 'LOADING', 'SYNCHING', 'READY', 'WORKING', 'ALL_CLAIMED', 'DONE', 'FAILED') DEFAULT 'READY' NOT NULL, */
total_job_count int(10) NOT NULL DEFAULT 0,
unclaimed_job_count int(10) NOT NULL DEFAULT 0,
done_job_count int(10) NOT NULL DEFAULT 0,
failed_job_count int(10) NOT NULL DEFAULT 0,
num_running_workers int(10) default 0 NOT NULL,
num_required_workers int(10) NOT NULL DEFAULT 0,
behaviour TEXT DEFAULT 'STATIC' NOT NULL, /* enum('STATIC', 'DYNAMIC') DEFAULT 'STATIC' NOT NULL, */
input_capacity int(10) default 4 NOT NULL,
output_capacity int(10) default 4 NOT NULL,
avg_msec_per_job int(10) default 0 NOT NULL,
avg_input_msec_per_job int(10) default 0 NOT NULL,
avg_run_msec_per_job int(10) default 0 NOT NULL,
avg_output_msec_per_job int(10) default 0 NOT NULL,
last_update datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
sync_lock int(10) default 0 NOT NULL
);
-- ---------------------------------------------------------------------------------
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment