Skip to content

GitLab

Commit 8f559390 authored by Leo Gordon's avatar Leo Gordon
Browse files

testing the idea of analyses that can be empty for blocking purposes (it does... 

testing the idea of analyses that can be empty for blocking purposes (it does not block while it is empty)
parent ea0d0271
Hide whitespace changes
Inline Side-by-side
Showing with 90 additions and 84 deletions
modules/Bio/EnsEMBL/Hive/AnalysisStats.pm
View file @ 8f559390
......@@ -252,41 +252,13 @@ sub rc_id {
return $self->{'_rc_id'};
}
sub determine_status {
my $self = shift;
if($self->status ne 'BLOCKED') {
if ($self->unclaimed_job_count == 0 and
$self->total_job_count == $self->done_job_count + $self->failed_job_count) {
my $failure_percentage = 0;
if ($self->total_job_count) {
$failure_percentage = $self->failed_job_count * 100 / $self->total_job_count;
}
if ($failure_percentage > $self->failed_job_tolerance) {
$self->status('FAILED');
print
"\n",
"##################################################\n",
printf
"## ERROR: %-35s ##\n", $self->get_analysis->logic_name." failed!";
printf
"## %4.1f%% jobs failed (tolerance: %3d%%) ##\n", $failure_percentage, $self->failed_job_tolerance;
print
"##################################################\n\n";
} else {
$self->status('DONE');
}
}
if($self->total_job_count == $self->unclaimed_job_count) {
$self->status('READY');
}
if($self->unclaimed_job_count>0 and
$self->total_job_count > $self->unclaimed_job_count) {
$self->status('WORKING');
}
}
return $self;
sub can_be_empty {
my $self = shift;
$self->{'_can_be_empty'} = shift if(@_);
return $self->{'_can_be_empty'};
}
sub print_stats {
my $self = shift;
......@@ -330,4 +302,79 @@ sub print_stats {
}
sub determine_status {
my $self = shift;
if($self->status ne 'BLOCKED') {
if ($self->unclaimed_job_count == 0 and
$self->total_job_count == $self->done_job_count + $self->failed_job_count) {
my $failure_percentage = 0;
if ($self->total_job_count) {
$failure_percentage = $self->failed_job_count * 100 / $self->total_job_count;
}
if ($failure_percentage > $self->failed_job_tolerance) {
$self->status('FAILED');
print
"\n",
"##################################################\n",
printf
"## ERROR: %-35s ##\n", $self->get_analysis->logic_name." failed!";
printf
"## %4.1f%% jobs failed (tolerance: %3d%%) ##\n", $failure_percentage, $self->failed_job_tolerance;
print
"##################################################\n\n";
} else {
$self->status('DONE');
}
}
if($self->total_job_count == $self->unclaimed_job_count) {
$self->status('READY');
}
if($self->unclaimed_job_count>0 and
$self->total_job_count > $self->unclaimed_job_count) {
$self->status('WORKING');
}
}
return $self;
}
sub check_blocking_control_rules {
my $self = shift;
my $ctrl_rules = $self->adaptor->db->get_AnalysisCtrlRuleAdaptor->fetch_by_ctrled_analysis_id($self->analysis_id);
if(scalar @$ctrl_rules) { # there are blocking ctrl_rules to check
my $all_ctrl_rules_done = 1;
foreach my $ctrl_rule (@$ctrl_rules) {
#use this method because the condition_analysis objects can be
#network distributed to a different database so use it's adaptor to get
#the AnalysisStats object
my $condition_analysis = $ctrl_rule->condition_analysis;
my $condition_analysis_stats = $condition_analysis && $condition_analysis->stats;
my $condition_analysis_stats_status = $condition_analysis_stats && $condition_analysis_stats->status;
my $condition_analysis_stats_cbe = $condition_analysis_stats && $condition_analysis_stats->can_be_empty;
unless( ($condition_analysis_stats_status eq 'DONE')
or ($condition_analysis_stats_cbe and ($condition_analysis_stats_status eq 'READY'))
) {
$all_ctrl_rules_done = 0;
last;
}
}
if($all_ctrl_rules_done) {
if($self->status eq 'BLOCKED') { # unblock, since all conditions are met
$self->update_status('LOADING'); # trigger sync
}
} else { # (re)block
$self->update_status('BLOCKED');
}
}
}
1;
modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisStatsAdaptor.pm
View file @ 8f559390
......@@ -212,6 +212,7 @@ sub update {
$sql .= ",last_update=NOW()";
$sql .= ",sync_lock='0'";
$sql .= ",rc_id=". $stats->rc_id();
$sql .= ",can_be_empty=". $stats->can_be_empty();
$sql .= " WHERE analysis_id='".$stats->analysis_id."' ";
my $sth = $self->prepare($sql);
......@@ -466,6 +467,7 @@ sub _columns {
ast.last_update
ast.sync_lock
ast.rc_id
ast.can_be_empty
);
push @columns , "UNIX_TIMESTAMP()-UNIX_TIMESTAMP(ast.last_update) seconds_since_last_update ";
return @columns;
......@@ -486,6 +488,7 @@ sub _objs_from_sth {
$analStats->status($column{'status'});
$analStats->sync_lock($column{'sync_lock'});
$analStats->rc_id($column{'rc_id'});
$analStats->can_be_empty($column{'can_be_empty'});
$analStats->batch_size($column{'batch_size'});
$analStats->avg_msec_per_job($column{'avg_msec_per_job'});
$analStats->avg_input_msec_per_job($column{'avg_input_msec_per_job'});
......
modules/Bio/EnsEMBL/Hive/PipeConfig/HiveGeneric_conf.pm
View file @ 8f559390
......@@ -344,8 +344,8 @@ sub run {
my $analysis_adaptor = $hive_dba->get_AnalysisAdaptor;
foreach my $aha (@{$self->pipeline_analyses}) {
my ($logic_name, $module, $parameters_hash, $input_ids, $program_file, $blocked, $batch_size, $hive_capacity, $failed_job_tolerance, $rc_id) =
rearrange([qw(logic_name module parameters input_ids program_file blocked batch_size hive_capacity failed_job_tolerance rc_id)], %$aha);
my ($logic_name, $module, $parameters_hash, $input_ids, $program_file, $blocked, $batch_size, $hive_capacity, $failed_job_tolerance, $can_be_empty, $rc_id) =
rearrange([qw(logic_name module parameters input_ids program_file blocked batch_size hive_capacity failed_job_tolerance can_be_empty rc_id)], %$aha);
$parameters_hash ||= {};
$input_ids ||= [];
......@@ -382,6 +382,7 @@ sub run {
$stats->hive_capacity( $hive_capacity ) if(defined($hive_capacity));
$stats->failed_job_tolerance( $failed_job_tolerance ) if(defined($failed_job_tolerance));
$stats->rc_id( $rc_id ) if(defined($rc_id));
$stats->can_be_empty( $can_be_empty ) if(defined($can_be_empty));
$stats->status($blocked ? 'BLOCKED' : 'READY'); # (some analyses will be waiting for human intervention in blocked state)
$stats->update();
}
......
modules/Bio/EnsEMBL/Hive/Queen.pm
View file @ 8f559390
......@@ -443,7 +443,7 @@ sub synchronize_hive {
print STDERR "Checking blocking control rules:\n";
foreach my $analysis (@$list_of_analyses) {
$self->check_blocking_control_rules_for_AnalysisStats($analysis->stats);
$analysis->stats->check_blocking_control_rules();
print STDERR '.';
}
print STDERR "\n";
......@@ -561,7 +561,7 @@ sub synchronize_AnalysisStats {
$analysisStats->done_job_count($done_here + $done_elsewhere);
$self->check_blocking_control_rules_for_AnalysisStats($analysisStats);
$analysisStats->check_blocking_control_rules();
if($analysisStats->status ne 'BLOCKED') {
$analysisStats->determine_status();
......@@ -589,43 +589,6 @@ sub synchronize_AnalysisStats {
}
sub check_blocking_control_rules_for_AnalysisStats
{
my $self = shift;
my $stats = shift;
return unless($stats);
#print("check ctrl on analysis "); $stats->print_stats;
my $ctrlRules = $self->db->get_AnalysisCtrlRuleAdaptor->
fetch_by_ctrled_analysis_id($stats->analysis_id);
my $allRulesDone = 1;
if(scalar @$ctrlRules > 0) {
#print("HAS blocking_ctrl_rules to check\n");
foreach my $ctrlrule (@{$ctrlRules}) {
#use this method because the condition_analysis objects can be
#network distributed to a different database so use it's adaptor to get
#the AnalysisStats object
#$ctrlrule->print_rule;
my $condAnalysis = $ctrlrule->condition_analysis;
my $condStats = $condAnalysis->stats if($condAnalysis);
$allRulesDone = 0 unless($condStats and $condStats->status eq 'DONE');
#print(" "); $condStats->print_stats;
}
if($allRulesDone) {
if($stats->status eq 'BLOCKED') {
#print(" UNBLOCK analysis : all conditions met\n");
$stats->update_status('LOADING'); #trigger sync
}
} else {
#print(" RE-BLOCK analysis : some conditions failed\n");
$stats->update_status('BLOCKED');
}
}
}
sub get_num_failed_analyses {
my ($self, $analysis) = @_;
......@@ -792,15 +755,6 @@ sub get_remaining_jobs_show_hive_progress {
return $remaining;
}
## Can't see where this method is used.
#
#sub print_hive_status {
# my ($self, $filter_analysis) = @_;
#
# $self->print_analysis_status($filter_analysis);
# $self->print_running_worker_status;
#}
sub print_analysis_status {
my ($self, $filter_analysis) = @_;
......
sql/tables.sql
View file @ 8f559390
......@@ -395,6 +395,7 @@ CREATE TABLE analysis_stats (
last_update datetime NOT NULL,
sync_lock int(10) default 0 NOT NULL,
rc_id int(10) unsigned default 0 NOT NULL,
can_be_empty TINYINT UNSIGNED DEFAULT 0 NOT NULL,
UNIQUE KEY (analysis_id),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment