Schema change: track the worker temp directories in the database

This allows cleaning-up the right directory

modules/Bio/EnsEMBL/Hive/Queen.pm

@@ -572,7 +572,6 @@ sub check_for_dead_workers {    # scans the whole Valley for lost Workers (but i
 
                     if( ($worker->status ne 'SUBMITTED')                 # There is no worker_temp_directory before specialization
                     and ($worker->meadow_user eq $this_meadow_user) ) {  # if I'm actually allowed to kill the worker...
-                            $worker->set_temp_directory_name( $this_meadow->config_get('BaseTempDirectory') );
                             $this_meadow->cleanup_temp_directory( $worker );
                     }
                 }

modules/Bio/EnsEMBL/Hive/Worker.pm

@@ -947,6 +947,7 @@ sub set_temp_directory_name {
     }
 
     $self->temp_directory_name( $temp_directory_name );
+    $self->adaptor->update_temp_directory_name( $self ) if $self->adaptor;  # autoloaded
 }
 
 

sql/patch_2019-05-10.mysql

+-- Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+-- Copyright [2016-2019] EMBL-European Bioinformatics Institute
+-- 
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+-- 
+--      http://www.apache.org/licenses/LICENSE-2.0
+-- 
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+-- ---------------------------------------------------------------------------------------------------
+
+SET @expected_version = 95;
+
+    -- make MySQL stop immediately after it encounters division by zero:
+SET SESSION sql_mode='TRADITIONAL';
+
+    -- warn that we detected the schema version mismatch:
+SELECT CONCAT(  'The patch only applies to schema version ',
+                @expected_version,
+                ', but the current schema version is ',
+                meta_value,
+                ', so skipping the rest.') AS ''
+    FROM hive_meta WHERE meta_key='hive_sql_schema_version' AND meta_value<>@expected_version;
+
+    -- cause division by zero only if current version differs from the expected one:
+INSERT INTO hive_meta (meta_key, meta_value)
+    SELECT 'this_should_never_be_inserted', 1 FROM hive_meta WHERE NOT 1/(meta_key<>'hive_sql_schema_version' OR meta_value=@expected_version);
+
+SELECT CONCAT(  'The patch seems to be compatible with schema version ',
+                @expected_version,
+                ', applying the patch...') AS '';
+
+    -- Now undo the change so that we could patch potentially non-TRADITIONAL schema:
+SET SESSION sql_mode='';
+
+-- ----------------------------------<actual_patch> -------------------------------------------------
+
+ALTER TABLE worker ADD COLUMN temp_directory_name VARCHAR(255) AFTER cause_of_death;
+
+-- ----------------------------------</actual_patch> -------------------------------------------------
+
+
+    -- increase the schema version by one and register the patch:
+UPDATE hive_meta SET meta_value=meta_value+1 WHERE meta_key='hive_sql_schema_version';
+INSERT INTO hive_meta (meta_key, meta_value) SELECT CONCAT("patched_to_", meta_value), CURRENT_TIMESTAMP FROM hive_meta WHERE meta_key = "hive_sql_schema_version";

sql/patch_2019-05-10.pgsql

+-- Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+-- Copyright [2016-2019] EMBL-European Bioinformatics Institute
+-- 
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+-- 
+--      http://www.apache.org/licenses/LICENSE-2.0
+-- 
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+-- ---------------------------------------------------------------------------------------------------
+
+\set expected_version 95
+
+\set ON_ERROR_STOP on
+
+    -- warn that we detected the schema version mismatch:
+SELECT ('The patch only applies to schema version '
+    || CAST(:expected_version AS VARCHAR)
+    || ', but the current schema version is '
+    || meta_value
+    || ', so skipping the rest.') as incompatible_msg
+    FROM hive_meta WHERE meta_key='hive_sql_schema_version' AND meta_value!=CAST(:expected_version AS VARCHAR);
+
+    -- cause division by zero only if current version differs from the expected one:
+INSERT INTO hive_meta (meta_key, meta_value)
+   SELECT 'this_should_never_be_inserted', 1 FROM hive_meta WHERE 1 != 1/CAST( (meta_key!='hive_sql_schema_version' OR meta_value=CAST(:expected_version AS VARCHAR)) AS INTEGER );
+
+SELECT ('The patch seems to be compatible with schema version '
+    || CAST(:expected_version AS VARCHAR)
+    || ', applying the patch...') AS compatible_msg;
+
+
+-- ----------------------------------<actual_patch> -------------------------------------------------
+
+ALTER TABLE worker ADD COLUMN temp_directory_name VARCHAR(255) DEFAULT NULL;
+
+-- ----------------------------------</actual_patch> -------------------------------------------------
+
+
+    -- increase the schema version by one and register the patch:
+UPDATE hive_meta SET meta_value= (CAST(meta_value AS INTEGER) + 1) WHERE meta_key='hive_sql_schema_version';
+INSERT INTO hive_meta (meta_key, meta_value) SELECT 'patched_to_' || meta_value, CURRENT_TIMESTAMP FROM hive_meta WHERE meta_key = 'hive_sql_schema_version';

sql/tables.mysql

@@ -497,6 +497,7 @@ CREATE TABLE analysis_data (
 @column when_seen           when the Worker was last seen by the Meadow
 @column when_died           if defined, when the Worker died (or its premature death was first detected by GC)
 @column cause_of_death      if defined, why did the Worker exit (or why it was killed)
+@column temp_directory_name a filesystem directory where this Worker can store temporary data
 @column log_dir             if defined, a filesystem directory where this Worker's output is logged
 */
 
@@ -518,6 +519,7 @@ CREATE TABLE worker (
     when_seen               TIMESTAMP                    NULL,
     when_died               TIMESTAMP                    NULL,      -- mysql's special for "TIMESTAMP DEFAULT NULL"
     cause_of_death          VARCHAR(255)         DEFAULT NULL,      -- expected values: 'NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'LIMBO', 'UNKNOWN'
+    temp_directory_name     VARCHAR(255)         DEFAULT NULL,
     log_dir                 VARCHAR(255)         DEFAULT NULL,
 
     KEY meadow_process (meadow_type, meadow_name, process_id)

sql/tables.pgsql

@@ -509,6 +509,7 @@ CREATE INDEX ON analysis_data (md5sum);
 @column when_seen           when the Worker was last seen by the Meadow
 @column when_died           if defined, when the Worker died (or its premature death was first detected by GC)
 @column cause_of_death      if defined, why did the Worker exit (or why it was killed)
+@column temp_directory_name a filesystem directory where this Worker can store temporary data
 @column log_dir             if defined, a filesystem directory where this Worker's output is logged
 */
 
@@ -529,6 +530,7 @@ CREATE TABLE worker (
     when_seen               TIMESTAMP            DEFAULT NULL,
     when_died               TIMESTAMP            DEFAULT NULL,
     cause_of_death          VARCHAR(255)         DEFAULT NULL,      -- expected values: 'NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'LIMBO', 'UNKNOWN'
+    temp_directory_name     VARCHAR(255)         DEFAULT NULL,
     log_dir                 VARCHAR(255)         DEFAULT NULL
 );
 CREATE INDEX ON worker (meadow_type, meadow_name, process_id);

sql/tables.sqlite

@@ -490,6 +490,7 @@ CREATE        INDEX analysis_data_idx ON analysis_data (md5sum);
 @column when_seen           when the Worker was last seen by the Meadow
 @column when_died           if defined, when the Worker died (or its premature death was first detected by GC)
 @column cause_of_death      if defined, why did the Worker exit (or why it was killed)
+@column temp_directory_name a filesystem directory where this Worker can store temporary data
 @column log_dir             if defined, a filesystem directory where this Worker's output is logged
 */
 
@@ -511,6 +512,7 @@ CREATE TABLE worker (
     when_seen               TIMESTAMP            DEFAULT NULL,
     when_died               TIMESTAMP            DEFAULT NULL,
     cause_of_death          VARCHAR(255)         DEFAULT NULL, /* enum('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'LIMBO', 'UNKNOWN') DEFAULT NULL */
+    temp_directory_name     VARCHAR(255)         DEFAULT NULL,
     log_dir                 VARCHAR(255)         DEFAULT NULL
 );
 CREATE INDEX worker_meadow_type_meadow_name_process_id_idx ON worker (meadow_type, meadow_name, process_id);

t/03.scripts/runWorker.t

+#!/usr/bin/env perl
+# Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# Copyright [2016-2019] EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+use strict;
+use warnings;
+
+use File::Temp qw{tempdir};
+use Test::Exception;
+use Test::More;
+
+use Bio::EnsEMBL::Hive::Utils::Test qw(init_pipeline get_test_url_or_die runWorker safe_drop_database);
+
+# eHive needs this to initialize the pipeline (and run db_cmd.pl)
+use Cwd            ();
+use File::Basename ();
+$ENV{'EHIVE_ROOT_DIR'} ||= File::Basename::dirname( File::Basename::dirname( File::Basename::dirname( Cwd::realpath($0) ) ) );
+
+my $pipeline_url = get_test_url_or_die;
+
+init_pipeline(
+    'Bio::EnsEMBL::Hive::Examples::LongMult::PipeConfig::LongMult_conf',
+    $pipeline_url,
+    [],
+    ['pipeline.param[take_time]=0', 'analysis[take_b_apart].meadow_type=undef', 'analysis[take_b_apart].analysis_capacity=1'],
+);
+
+my $dir = tempdir CLEANUP => 1;
+
+runWorker($pipeline_url, [ -job_id => 1, ]);
+runWorker($pipeline_url, [ -job_id => 2, -worker_base_temp_dir => $dir ]);
+
+my $hive_dba = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new( -url => $pipeline_url );
+
+my $worker1 = $hive_dba->get_WorkerAdaptor->fetch_by_dbID(1);
+my $worker2 = $hive_dba->get_WorkerAdaptor->fetch_by_dbID(2);
+
+unlike($worker1->temp_directory_name, qr{^$dir}, "The first worker is not using $dir");
+  like($worker2->temp_directory_name, qr{^$dir}, "The second worker used $dir");
+
+safe_drop_database( $hive_dba );
+
+done_testing();
+