schema_change: preregister Workers upon submission in SUBMITTED state, added...

schema_change: preregister Workers upon submission in SUBMITTED state, added when_submitted, relaxed cause_of_death to VARCHAR

modules/Bio/EnsEMBL/Hive/Meadow/LOCAL.pm

@@ -117,7 +117,7 @@ sub kill_worker {
 }
 
 
-sub submit_workers {
+sub submit_workers_return_meadow_pids {
     my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_log_subdir) = @_;
 
     my @worker_cmd_components = split_for_bash($worker_cmd);  # FIXME: change the interface so that $worker_cmd itself is passed in as ARRAYref

modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm

@@ -299,7 +299,7 @@ sub get_report_entries_for_time_interval {
 }
 
 
-sub submit_workers {
+sub submit_workers_return_meadow_pids {
     my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_log_subdir) = @_;
 
     my $job_array_common_name               = $self->job_array_common_name($rc_name, $iteration);

modules/Bio/EnsEMBL/Hive/Queen.pm

@@ -112,9 +112,9 @@ sub create_new_worker {
     my $self    = shift @_;
     my %flags   = @_;
 
-    my ($resource_class_id, $resource_class_name, $beekeeper_id,
+    my ($preregistered, $resource_class_id, $resource_class_name, $beekeeper_id,
         $no_write, $debug, $worker_log_dir, $hive_log_dir, $job_limit, $life_span, $no_cleanup, $retry_throwing_jobs, $can_respecialize)
-     = @flags{qw(-resource_class_id -resource_class_name -beekeeper_id
+     = @flags{qw(-preregistered -resource_class_id -resource_class_name -beekeeper_id
             -no_write -debug -worker_log_dir -hive_log_dir -job_limit -life_span -no_cleanup -retry_throwing_jobs -can_respecialize)};
 
     my ($meadow, $process_id, $meadow_host, $meadow_user) = Bio::EnsEMBL::Hive::Valley->new()->whereami();
@@ -123,7 +123,7 @@ sub create_new_worker {
     my $meadow_name = $meadow->cached_name;
 
     foreach my $prev_worker_incarnation (@{ $self->find_previous_worker_incarnations($meadow_type, $meadow_name, $process_id) }) {
-            # so far 'RELOCATED events' has been detected on LSF 9.0 in response to sending signal #99 or #100
+            # So far 'RELOCATED events' has been detected on LSF 9.0 in response to sending signal #99 or #100
             # Since I don't know how to avoid them, I am trying to register them when they happen.
             # The following snippet buries the previous incarnation of the Worker before starting a new one.
             #
@@ -134,31 +134,55 @@ sub create_new_worker {
         $self->register_worker_death( $prev_worker_incarnation );
     }
 
-    my $resource_class;
+    my $worker;
 
-    if( defined($resource_class_name) ) {
-        $resource_class = $self->db->get_ResourceClassAdaptor->fetch_by_name($resource_class_name)
-            or die "resource_class with name='$resource_class_name' could not be fetched from the database";
-    } elsif( defined($resource_class_id) ) {
-        $resource_class = $self->db->get_ResourceClassAdaptor->fetch_by_dbID($resource_class_id)
-            or die "resource_class with dbID='$resource_class_id' could not be fetched from the database";
-    }
+    if($preregistered) {
 
-    my $worker = Bio::EnsEMBL::Hive::Worker->new(
-        'meadow_type'       => $meadow_type,
-        'meadow_name'       => $meadow_name,
-        'meadow_host'       => $meadow_host,
-        'meadow_user'       => $meadow_user,
-        'process_id'        => $process_id,
-        'resource_class'    => $resource_class,
-        'beekeeper_id'      => $beekeeper_id,
-    );
+        my $sec = 1;
+        until( $worker = $self->fetch_preregistered_worker($meadow_type, $meadow_name, $process_id) ) {
+            $self->db->get_LogMessageAdaptor->store_hive_message("Preregistered Worker $meadow_type/$meadow_name:$process_id waiting $sec more seconds to fetch itself...", 'WORKER_CAUTION' );
+            sleep($sec);
+            $sec *= 2;
+        }
+
+            # only update the fields that were not available at the time of submission:
+        $worker->meadow_host( $meadow_host );
+        $worker->meadow_user( $meadow_user );
+        $worker->when_born(   'CURRENT_TIMESTAMP' );
+        $worker->status(      'READY' );
+
+        $self->update( $worker );
+
+    } else {
+        my $resource_class;
+
+        if( defined($resource_class_name) ) {
+            $resource_class = $self->db->get_ResourceClassAdaptor->fetch_by_name($resource_class_name)
+                or die "resource_class with name='$resource_class_name' could not be fetched from the database";
+        } elsif( defined($resource_class_id) ) {
+            $resource_class = $self->db->get_ResourceClassAdaptor->fetch_by_dbID($resource_class_id)
+                or die "resource_class with dbID='$resource_class_id' could not be fetched from the database";
+        }
+
+        $worker = Bio::EnsEMBL::Hive::Worker->new(
+            'meadow_type'       => $meadow_type,
+            'meadow_name'       => $meadow_name,
+            'process_id'        => $process_id,
+            'resource_class'    => $resource_class,
+            'beekeeper_id'      => $beekeeper_id,
+
+            'meadow_host'       => $meadow_host,
+            'meadow_user'       => $meadow_user,
+        );
+
+        if (ref($self)) {
+            $self->store( $worker );
+
+            $worker->when_born(   'CURRENT_TIMESTAMP' );
+            $self->update_when_born( $worker );
 
-    if (ref($self)) {
-        $self->store( $worker );
-        my $worker_id = $worker->dbID;
-        $worker = $self->fetch_by_dbID( $worker_id )    # refresh the object to get the fields initialized at SQL level (timestamps in this case)
-            or die "Could not fetch worker with dbID=$worker_id";
+            $self->refresh( $worker );
+        }
     }
 
     $worker->set_log_directory_name($hive_log_dir, $worker_log_dir);
@@ -475,7 +499,19 @@ sub find_previous_worker_incarnations {
     # This happens in standalone mode, when there is no database
     return [] unless ref($self);
 
-    return $self->fetch_all( "status!='DEAD' AND meadow_type='$meadow_type' AND meadow_name='$meadow_name' AND process_id='$process_id'" );
+    return $self->fetch_all( "status!='DEAD' AND status!='SUBMITTED' AND meadow_type='$meadow_type' AND meadow_name='$meadow_name' AND process_id='$process_id'" );
+}
+
+
+sub fetch_preregistered_worker {
+    my ($self, $meadow_type, $meadow_name, $process_id) = @_;
+
+    # This happens in standalone mode, when there is no database
+    return [] unless ref($self);
+
+    my ($worker) = @{ $self->fetch_all( "status='SUBMITTED' AND meadow_type='$meadow_type' AND meadow_name='$meadow_name' AND process_id='$process_id'" ) };
+
+    return $worker;
 }
 
 

modules/Bio/EnsEMBL/Hive/Scripts/RunWorker.pm

@@ -52,6 +52,8 @@ sub runWorker {
 
     # Create the worker
     my $worker = $queen->create_new_worker(
+             -preregistered         => $specialization_options->{'preregistered'},
+
           # Resource class:
              -resource_class_id     => $specialization_options->{'resource_class_id'},
              -resource_class_name   => $specialization_options->{'resource_class_name'},

modules/Bio/EnsEMBL/Hive/Worker.pm

@@ -166,12 +166,21 @@ sub status {
     return $self->{'_status'};
 }
 
+
 sub beekeeper_id {
     my $self = shift;
     $self->{'_beekeeper_id'} = shift if(@_);
     return $self->{'_beekeeper_id'} || undef;
 }
 
+
+sub when_submitted {
+    my $self = shift;
+    $self->{'_when_submitted'} = shift if(@_);
+    return $self->{'_when_submitted'};
+}
+
+
 sub when_born {
     my $self = shift;
     $self->{'_when_born'} = shift if(@_);

scripts/beekeeper.pl

@@ -426,7 +426,7 @@ sub generate_worker_cmd {
         exit(1);
     }
 
-    foreach my $worker_option ('url', 'reg_conf', 'reg_type', 'reg_alias', 'nosqlvc', 'beekeeper_id', 'job_limit', 'life_span', 'retry_throwing_jobs', 'can_respecialize', 'hive_log_dir', 'debug') {
+    foreach my $worker_option ('url', 'reg_conf', 'reg_type', 'reg_alias', 'nosqlvc', 'job_limit', 'life_span', 'retry_throwing_jobs', 'can_respecialize', 'hive_log_dir', 'debug') {
         if(defined(my $value = $self->{$worker_option})) {
             $worker_cmd .= " -${worker_option} $value";
         }
@@ -572,17 +572,42 @@ sub run_autonomously {
                         'INFO', 'ALIVE');
 
                     my ($submission_cmd_args, $worker_cmd_args) = @{ $meadow_type_rc_name2resource_param_list{ $meadow_type }{ $rc_name } || [] };
+                    my $meadow_can_preregister_workers          = UNIVERSAL::can($this_meadow, 'submit_workers_return_meadow_pids');
 
                     my $specific_worker_cmd = $resourceless_worker_cmd
-                                            . " -rc_name $rc_name"
-                                            . (defined($worker_cmd_args) ? " $worker_cmd_args" : '');
+                                            . (defined($worker_cmd_args) ? " $worker_cmd_args" : '')
+                                            . ( $meadow_can_preregister_workers
+                                                    ? ' -preregistered'
+                                                    : ' -beekeeper_id '.$self->{'beekeeper_id'}." -rc_name $rc_name" );
 
+                        # Since the newer version of Meadow interface can parse and return $meadow_process_ids, the Workers can be preregistered in the database:
+                    if($meadow_can_preregister_workers) {
 
-                    if( my $meadow_process_ids = $this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration,
-                                                    $rc_name, $submission_cmd_args || '', $submit_log_subdir) ) {
-                        # Since the newer version of Meadow interface can parse and return $meadow_process_ids, the Workers can be pre-registered in the database
+                        my $meadow_process_ids = $this_meadow->submit_workers_return_meadow_pids(
+                            $specific_worker_cmd, $this_meadow_rc_worker_count, $iteration, $rc_name, $submission_cmd_args || '', $submit_log_subdir);
 
                         warn "Submitted the following process_ids to ".$this_meadow->signature.": ".join(', ', @$meadow_process_ids)."\n";
+
+                        my $resource_class  = $pipeline->collection_of('ResourceClass')->find_one_by('name', $rc_name);
+                        my $meadow_name     = $this_meadow->cached_name;
+
+                        my @pre_allocated_workers = map {
+                                Bio::EnsEMBL::Hive::Worker->new(
+                                    'meadow_type'       => $meadow_type,                # non-unique key components
+                                    'meadow_name'       => $meadow_name,
+                                    'process_id'        => $_,
+
+                                    'resource_class'    => $resource_class,             # non-key, but known at the time of pre-allocation
+                                    'beekeeper_id'      => $self->{'beekeeper_id'},
+
+                                    'status'            => 'SUBMITTED',
+                                )
+                        } @$meadow_process_ids;
+
+                        $queen->store( \@pre_allocated_workers );
+
+                    } else {
+                        $this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration, $rc_name, $submission_cmd_args || '', $submit_log_subdir);
                     }
                 }
             }

scripts/runWorker.pl

@@ -24,7 +24,7 @@ main();
 
 sub main {
     my ($url, $reg_conf, $reg_type, $reg_alias, $nosqlvc);                   # Connection parameters
-    my ($resource_class_id, $resource_class_name, $analyses_pattern, $analysis_id, $logic_name, $job_id, $force, $beekeeper_id);  # Task specification parameters
+    my ($preregistered, $resource_class_id, $resource_class_name, $analyses_pattern, $analysis_id, $logic_name, $job_id, $force, $beekeeper_id);  # Task specification parameters
     my ($job_limit, $life_span, $no_cleanup, $no_write, $hive_log_dir, $worker_log_dir, $retry_throwing_jobs, $can_respecialize);   # Worker control parameters
     my ($help, $report_versions, $debug);
 
@@ -38,6 +38,7 @@ sub main {
                'nosqlvc=i'                    => \$nosqlvc,       # can't use the binary "!" as it is a propagated option
 
     # Task specification parameters:
+               'preregistered!'             => \$preregistered,
                'rc_id=i'                    => \$resource_class_id,
                'rc_name=s'                  => \$resource_class_name,
                'analyses_pattern=s'         => \$analyses_pattern,
@@ -116,6 +117,7 @@ sub main {
     }
 
     my %specialization_options = (
+        preregistered       => $preregistered,
         resource_class_id   => $resource_class_id,
         resource_class_name => $resource_class_name,
         can_respecialize    => $can_respecialize,

sql/patch_2017-05-11.mysql

+
+-- ---------------------------------------------------------------------------------------------------
+
+SET @expected_version = 91;
+
+    -- make MySQL stop immediately after it encounters division by zero:
+SET SESSION sql_mode='TRADITIONAL';
+
+    -- warn that we detected the schema version mismatch:
+SELECT CONCAT(  'The patch only applies to schema version ',
+                @expected_version,
+                ', but the current schema version is ',
+                meta_value,
+                ', so skipping the rest.') AS ''
+    FROM hive_meta WHERE meta_key='hive_sql_schema_version' AND meta_value<>@expected_version;
+
+    -- cause division by zero only if current version differs from the expected one:
+INSERT INTO hive_meta (meta_key, meta_value)
+    SELECT 'this_should_never_be_inserted', 1 FROM hive_meta WHERE NOT 1/(meta_key<>'hive_sql_schema_version' OR meta_value=@expected_version);
+
+SELECT CONCAT(  'The patch seems to be compatible with schema version ',
+                @expected_version,
+                ', applying the patch...') AS '';
+
+    -- Now undo the change so that we could patch potentially non-TRADITIONAL schema:
+SET SESSION sql_mode='';
+
+-- ----------------------------------<actual_patch> -------------------------------------------------
+
+
+-- part_1: changes to introduce 'SUBMITTED' state of Workers:
+
+ALTER TABLE worker      MODIFY COLUMN   meadow_host     VARCHAR(255)        DEFAULT NULL;     -- allow it to be NULL for 'SUBMITTED' entries
+
+ALTER TABLE worker      MODIFY COLUMN   when_born       TIMESTAMP                   NULL;
+
+ALTER TABLE worker      ADD COLUMN      when_submitted  TIMESTAMP  NOT NULL DEFAULT CURRENT_TIMESTAMP   AFTER beekeeper_id;
+
+
+-- part_2: changes to allow extra meadow-specific exit statuses to be representable in the worker table:
+
+ALTER TABLE worker      MODIFY COLUMN   cause_of_death  VARCHAR(255)        DEFAULT NULL;
+
+
+-- ----------------------------------</actual_patch> -------------------------------------------------
+
+
+    -- increase the schema version by one and register the patch:
+UPDATE hive_meta SET meta_value=meta_value+1 WHERE meta_key='hive_sql_schema_version';
+INSERT INTO hive_meta (meta_key, meta_value) SELECT CONCAT("patched_to_", meta_value), CURRENT_TIMESTAMP FROM hive_meta WHERE meta_key = "hive_sql_schema_version";

sql/patch_2017-05-11.pgsql

+
+-- ---------------------------------------------------------------------------------------------------
+
+\set expected_version 91
+
+\set ON_ERROR_STOP on
+
+    -- warn that we detected the schema version mismatch:
+SELECT ('The patch only applies to schema version '
+    || CAST(:expected_version AS VARCHAR)
+    || ', but the current schema version is '
+    || meta_value
+    || ', so skipping the rest.') as incompatible_msg
+    FROM hive_meta WHERE meta_key='hive_sql_schema_version' AND meta_value!=CAST(:expected_version AS VARCHAR);
+
+    -- cause division by zero only if current version differs from the expected one:
+INSERT INTO hive_meta (meta_key, meta_value)
+   SELECT 'this_should_never_be_inserted', 1 FROM hive_meta WHERE 1 != 1/CAST( (meta_key!='hive_sql_schema_version' OR meta_value=CAST(:expected_version AS VARCHAR)) AS INTEGER );
+
+SELECT ('The patch seems to be compatible with schema version '
+    || CAST(:expected_version AS VARCHAR)
+    || ', applying the patch...') AS compatible_msg;
+
+
+-- ----------------------------------<actual_patch> -------------------------------------------------
+
+-- part_1: changes to introduce 'SUBMITTED' state of Workers:
+
+ALTER TABLE worker      ALTER COLUMN    meadow_host     DROP NOT NULL;
+ALTER TABLE worker      ALTER COLUMN    meadow_host     SET DEFAULT NULL;
+
+ALTER TABLE worker      ALTER COLUMN    when_born       DROP NOT NULL;
+ALTER TABLE worker      ALTER COLUMN    when_born       SET DEFAULT NULL;
+
+ALTER TABLE worker      ADD COLUMN      when_submitted  TIMESTAMP  NOT NULL DEFAULT CURRENT_TIMESTAMP;
+
+-- part_2: changes to allow extra meadow-specific exit statuses to be representable in the worker table:
+
+ALTER TABLE worker      ALTER COLUMN   cause_of_death  SET DATA TYPE   VARCHAR(255);
+
+-- ----------------------------------</actual_patch> -------------------------------------------------
+
+
+    -- increase the schema version by one and register the patch:
+UPDATE hive_meta SET meta_value= (CAST(meta_value AS INTEGER) + 1) WHERE meta_key='hive_sql_schema_version';
+INSERT INTO hive_meta (meta_key, meta_value) SELECT 'patched_to_' || meta_value, CURRENT_TIMESTAMP FROM hive_meta WHERE meta_key = 'hive_sql_schema_version';

sql/tables.mysql

@@ -493,6 +493,7 @@ CREATE TABLE analysis_data (
 @column work_done           how many jobs the Worker has completed successfully
 @column status              current status of the Worker
 @column beekeeper_id        beekeeper that created this worker
+@column when_submitted      when the Worker was submitted by a Beekeeper
 @column when_born           when the Worker process was started
 @column when_checked_in     when the Worker last checked into the database
 @column when_seen           when the Worker was last seen by the Meadow
@@ -505,19 +506,20 @@ CREATE TABLE worker (
     worker_id               INTEGER     NOT NULL PRIMARY KEY AUTO_INCREMENT,
     meadow_type             VARCHAR(255) NOT NULL,
     meadow_name             VARCHAR(255) NOT NULL,
-    meadow_host             VARCHAR(255) NOT NULL,
+    meadow_host             VARCHAR(255)         DEFAULT NULL,
     meadow_user             VARCHAR(255)         DEFAULT NULL,
     process_id              VARCHAR(255) NOT NULL,
     resource_class_id       INTEGER              DEFAULT NULL,
 
     work_done               INTEGER      NOT NULL DEFAULT 0,
-    status                  VARCHAR(255) NOT NULL DEFAULT 'READY',  -- expected values: 'SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD'
+    status                  VARCHAR(255) NOT NULL DEFAULT 'READY',  -- expected values: 'SUBMITTED','SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD'
     beekeeper_id            INTEGER      DEFAULT NULL,
-    when_born               TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    when_submitted          TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    when_born               TIMESTAMP                    NULL,
     when_checked_in         TIMESTAMP                    NULL,
     when_seen               TIMESTAMP                    NULL,
-    when_died               TIMESTAMP                    NULL,  -- mysql's special for "TIMESTAMP DEFAULT NULL"
-    cause_of_death          ENUM('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN') DEFAULT NULL,
+    when_died               TIMESTAMP                    NULL,      -- mysql's special for "TIMESTAMP DEFAULT NULL"
+    cause_of_death          VARCHAR(255)         DEFAULT NULL,      -- expected values: 'NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN'
     log_dir                 VARCHAR(255)         DEFAULT NULL,
 
     KEY meadow_process (meadow_type, meadow_name, process_id)

sql/tables.pgsql

@@ -506,6 +506,7 @@ CREATE INDEX ON analysis_data (md5sum);
 @column work_done           how many jobs the Worker has completed successfully
 @column status              current status of the Worker
 @column beekeeper_id        beekeeper that created this worker
+@column when_submitted      when the Worker was submitted by a Beekeeper
 @column when_born           when the Worker process was started
 @column when_checked_in     when the Worker last checked into the database
 @column when_seen           when the Worker was last seen by the Meadow
@@ -519,18 +520,19 @@ CREATE TABLE worker (
     worker_id               SERIAL PRIMARY KEY,
     meadow_type             VARCHAR(255) NOT NULL,
     meadow_name             VARCHAR(255) NOT NULL,
-    meadow_host             VARCHAR(255) NOT NULL,
+    meadow_host             VARCHAR(255)         DEFAULT NULL,
     meadow_user             VARCHAR(255)         DEFAULT NULL,
     process_id              VARCHAR(255) NOT NULL,
     resource_class_id       INTEGER              DEFAULT NULL,
     work_done               INTEGER      NOT NULL DEFAULT 0,
-    status                  VARCHAR(255) NOT NULL DEFAULT 'READY',  -- expected values: 'SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD'
+    status                  VARCHAR(255) NOT NULL DEFAULT 'READY',  -- expected values: 'SUBMITTED','SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD'
     beekeeper_id            INTEGER      DEFAULT NULL,
-    when_born               TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    when_submitted          TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    when_born               TIMESTAMP            DEFAULT NULL,
     when_checked_in         TIMESTAMP            DEFAULT NULL,
     when_seen               TIMESTAMP            DEFAULT NULL,
     when_died               TIMESTAMP            DEFAULT NULL,
-    cause_of_death          worker_cod           DEFAULT NULL,
+    cause_of_death          VARCHAR(255)         DEFAULT NULL,      -- expected values: 'NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN'
     log_dir                 VARCHAR(255)         DEFAULT NULL
 );
 CREATE INDEX ON worker (meadow_type, meadow_name, process_id);

sql/tables.sqlite

@@ -487,6 +487,7 @@ CREATE        INDEX analysis_data_idx ON analysis_data (md5sum);
 @column work_done           how many jobs the Worker has completed successfully
 @column status              current status of the Worker
 @column beekeeper_id        beekeeper that created this worker
+@column when_submitted      when the Worker was submitted by a Beekeeper
 @column when_born           when the Worker process was started
 @column when_checked_in     when the Worker last checked into the database
 @column when_seen           when the Worker was last seen by the Meadow
@@ -499,18 +500,19 @@ CREATE TABLE worker (
     worker_id               INTEGER     NOT NULL PRIMARY KEY AUTOINCREMENT,
     meadow_type             VARCHAR(255) NOT NULL,
     meadow_name             VARCHAR(255) NOT NULL,
-    meadow_host             VARCHAR(255) NOT NULL,
+    meadow_host             VARCHAR(255)         DEFAULT NULL,
     meadow_user             VARCHAR(255)         DEFAULT NULL,
     process_id              VARCHAR(255) NOT NULL,
     resource_class_id       INTEGER              DEFAULT NULL,
     work_done               INTEGER      NOT NULL DEFAULT 0,
-    status                  VARCHAR(255) NOT NULL DEFAULT 'READY', /* enum('SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD') DEFAULT 'READY' NOT NULL */
+    status                  VARCHAR(255) NOT NULL DEFAULT 'READY', /* enum('SUBMITTED','SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD') DEFAULT 'READY' NOT NULL */
     beekeeper_id            INTEGER      DEFAULT NULL,
-    when_born               TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    when_submitted          TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    when_born               TIMESTAMP            DEFAULT NULL,
     when_checked_in         TIMESTAMP            DEFAULT NULL,
     when_seen               TIMESTAMP            DEFAULT NULL,
     when_died               TIMESTAMP            DEFAULT NULL,
-    cause_of_death          TEXT                 DEFAULT NULL, /* enum('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN') DEFAULT NULL */
+    cause_of_death          VARCHAR(255)         DEFAULT NULL, /* enum('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN') DEFAULT NULL */
     log_dir                 VARCHAR(255)         DEFAULT NULL
 );
 CREATE INDEX worker_meadow_type_meadow_name_process_id_idx ON worker (meadow_type, meadow_name, process_id);

t/04.meadow/lsf.t

@@ -144,17 +144,17 @@ my $worker = Bio::EnsEMBL::Hive::Worker->new();
 
 lives_ok( sub {
     local $ENV{EHIVE_EXPECTED_BSUB} = '-o /dev/null -e /dev/null -J tracking_homo_sapiens_funcgen_81_38_hive-Hive-/resource_class/-56 /rc_args/ /worker_cmd/';
-    $lsf_meadow->submit_workers('/worker_cmd/', 1, 56, '/resource_class/', '/rc_args/');
+    $lsf_meadow->submit_workers_return_meadow_pids('/worker_cmd/', 1, 56, '/resource_class/', '/rc_args/');
 }, 'Can submit 1 worker');
 
 lives_ok( sub {
     local $ENV{EHIVE_EXPECTED_BSUB} = '-o /dev/null -e /dev/null -J tracking_homo_sapiens_funcgen_81_38_hive-Hive-/resource_class/-56[1-4] /rc_args/ /worker_cmd/';
-    $lsf_meadow->submit_workers('/worker_cmd/', 4, 56, '/resource_class/', '/rc_args/');
+    $lsf_meadow->submit_workers_return_meadow_pids('/worker_cmd/', 4, 56, '/resource_class/', '/rc_args/');
 }, 'Can submit 4 workers');
 
 lives_ok( sub {
     local $ENV{EHIVE_EXPECTED_BSUB} = '-o /submit_log_dir//log_/resource_class/_%J_%I.out -e /submit_log_dir//log_/resource_class/_%J_%I.err -J tracking_homo_sapiens_funcgen_81_38_hive-Hive-/resource_class/-56 /rc_args/ /worker_cmd/';
-    $lsf_meadow->submit_workers('/worker_cmd/', 1, 56, '/resource_class/', '/rc_args/', '/submit_log_dir/');
+    $lsf_meadow->submit_workers_return_meadow_pids('/worker_cmd/', 1, 56, '/resource_class/', '/rc_args/', '/submit_log_dir/');
 }, 'Can submit 1 worker with a submit_log_dir');
 
 my $expected_bacct = {