From 0d3c6612d0afebab9410ce2bf78b4f9a4360d82a Mon Sep 17 00:00:00 2001
From: Leo Gordon <lg4@ebi.ac.uk>
Date: Tue, 8 Jul 2014 17:29:32 +0100
Subject: [PATCH] balance_semaphores() returns the number of unsemaphored jobs,
and the second sync() is only done if a nonzero number of semaphored jobs
were unsemaphored
---
modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisJobAdaptor.pm | 5 +++++
modules/Bio/EnsEMBL/Hive/Scheduler.pm | 11 +++++++----
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisJobAdaptor.pm b/modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisJobAdaptor.pm
index cbfcd761c..853926efe 100644
--- a/modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisJobAdaptor.pm
+++ b/modules/Bio/EnsEMBL/Hive/DBSQL/AnalysisJobAdaptor.pm
@@ -608,6 +608,8 @@ sub balance_semaphores {
my $find_sth = $self->prepare($find_sql);
my $update_sth = $self->prepare($update_sql);
+ my $rebalanced_jobs_counter = 0;
+
$find_sth->execute();
while(my ($job_id, $was, $should) = $find_sth->fetchrow_array()) {
my $msg;
@@ -618,11 +620,14 @@ sub balance_semaphores {
$update_sth->execute($should-$was, $job_id);
$msg = "Semaphore count needed rebalancing now, so performing: $was -> $should";
$self->db->get_LogMessageAdaptor->store_job_message( $job_id, $msg, 1 );
+ $rebalanced_jobs_counter++;
}
warn "[Job $job_id] $msg\n"; # TODO: integrate the STDERR diagnostic output with LogMessageAdaptor calls in general
}
$find_sth->finish;
$update_sth->finish;
+
+ return $rebalanced_jobs_counter;
}
diff --git a/modules/Bio/EnsEMBL/Hive/Scheduler.pm b/modules/Bio/EnsEMBL/Hive/Scheduler.pm
index 586397766..d34ac4c23 100644
--- a/modules/Bio/EnsEMBL/Hive/Scheduler.pm
+++ b/modules/Bio/EnsEMBL/Hive/Scheduler.pm
@@ -67,16 +67,19 @@ sub schedule_workers_resync_if_necessary {
$queen->check_for_dead_workers($valley, 1);
}
+ print "Scheduler: re-synchronizing the Hive...\n";
+ $queen->synchronize_hive($filter_analysis);
+
if($queen->db->hive_auto_rebalance_semaphores) { # make sure rebalancing only ever happens for the pipelines that asked for it
print "Scheduler: re-balancing of semaphore_counts...\n";
- $queen->db->get_AnalysisJobAdaptor->balance_semaphores($filter_analysis && $filter_analysis->dbID);
+ if( my $rebalanced_jobs_counter = $queen->db->get_AnalysisJobAdaptor->balance_semaphores($filter_analysis && $filter_analysis->dbID) ) {
+ print "Scheduler: re-balanced $rebalanced_jobs_counter jobs, going through another re-synchronization of the Hive...\n";
+ $queen->synchronize_hive($filter_analysis);
+ }
} else {
print "Scheduler: automatic re-balancing of semaphore_counts is off by default. If you think your pipeline might benefit from it, set hive_auto_rebalance_semaphores => 1 in the PipeConfig's hive_meta_table.\n";
}
- print "Scheduler: re-synchronizing the Hive...\n";
- $queen->synchronize_hive($filter_analysis);
-
($workers_to_submit_by_meadow_type_rc_name, $total_extra_workers_required, $log_buffer)
= schedule_workers($queen, $submit_capacity, $default_meadow_type, undef, undef, $filter_analysis, $meadow_capacity_limiter_hashed_by_type, $analysis_id2rc_name);
print $log_buffer;
--
GitLab