avoiding RELOCATED workers by applying incemental backoff-and-retry approach...

avoiding RELOCATED workers by applying incemental backoff-and-retry approach from Ethernet CSMA/CD protocol in dbc::connect()

modules/Bio/EnsEMBL/Hive/DBSQL/DBConnection.pm

@@ -114,16 +114,53 @@ sub url {
 }
 
 
-sub protected_prepare_execute {     # try to resolve certain mysql "Deadlocks" by trying again (a useful workaround even in mysql 5.1.61)
+sub connect {       # a wrapper that imitates CSMA/CD protocol's incremental backoff-and-retry approach
+    my $self        = shift @_;
+
+    my $attempts    = 8;
+    my $sleep_sec   = 1;
+    my $retval;
+
+    foreach my $attempt (1..$attempts) {
+        eval {
+            $retval = $self->SUPER::connect( @_ );
+            1;
+        } or do {
+            if( ($@ =~ /Could not connect to database.+?failed: Too many connections/s)                             # problem on server side (configured with not enough connections)
+             or ($@ =~ /Could not connect to database.+?failed: Can't connect to \w+? server on '.+?' \(99\)/s)     # problem on client side (cooling down period after a disconnect)
+            ) {
+
+                warn "Possibly transient problem conecting to the database (attempt #$attempt). Will try again in $sleep_sec sec";
+
+                usleep( $sleep_sec*1000000 );
+                $sleep_sec *= 2;
+                next;
+
+            } else {     # but definitely report other errors
+
+                die $@;
+            }
+        };
+        last;   # stop looping once we succeeded
+    }
+
+    if($@) {
+        die "After $attempts attempts still could not connect() : $@";
+    }
+
+    return $retval;
+}
+
+
+sub protected_prepare_execute {     # try to resolve certain mysql "Deadlocks" by imitating CSMA/CD protocol's incremental backoff-and-retry approach (a useful workaround even in mysql 5.1.61)
     my $self                    = shift @_;
     my $sql_params              = shift @_;
     my $deadlock_log_callback   = shift @_;
 
-    my $sql_cmd     = shift @$sql_params;
+    my $sql_cmd         = shift @$sql_params;
 
     my $attempts        = 9;
     my $sleep_max_sec   = 1;
-    my $log_message_adaptor;
 
     my $retval;
     my $query_msg;
@@ -137,18 +174,12 @@ sub protected_prepare_execute {     # try to resolve certain mysql "Deadlocks" b
         } or do {
             $query_msg = "QUERY: $sql_cmd, PARAMS: (".join(', ',@$sql_params).")";
 
-            if( (my $deadlock_detected = ($@ =~ /Deadlock found when trying to get lock; try restarting transaction/))
-             or (my $toomanyconn_detected = ($@ =~ /Could not connect to database.+?failed: Too many connections/))
-            ) {
+            if( $@ =~ /Deadlock found when trying to get lock; try restarting transaction/ ) {
 
                 my $this_sleep_sec = int( rand( $sleep_max_sec )*100 ) / 100.0;
 
-                if($toomanyconn_detected) {     # It may get noticed in case the error log is monitored/recorded:
-                    warn "Too many connections detected when trying to run $query_msg (attempt #$attempt). Will try again in $this_sleep_sec sec";
-                } else {
-                    if( $deadlock_log_callback ) {
-                        $deadlock_log_callback->( " temporarily failed due to a DEADLOCK in the database (attempt #$attempt). Will try again in $this_sleep_sec sec" );
-                    }
+                if( $deadlock_log_callback ) {
+                    $deadlock_log_callback->( " temporarily failed due to a DEADLOCK in the database (attempt #$attempt). Will try again in $this_sleep_sec sec" );
                 }
 
                 usleep( $this_sleep_sec*1000000 );