ResourceDescription expanded to include both submission_cmd_args and...

ResourceDescription expanded to include both submission_cmd_args and worker_cmd_args. Both args can be specified in a PipeConfig file.

modules/Bio/EnsEMBL/Hive/Meadow/LOCAL.pm

@@ -94,12 +94,12 @@ sub kill_worker {
 
 
 sub submit_workers {
-    my ($self, $worker_cmd, $worker_count, $iteration, $rc_name, $rc_parameters) = @_;
+    my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_stdout_file, $submit_stderr_file) = @_;
 
     my $cmd = "$worker_cmd &";
 
-    foreach (1..$worker_count) {
-        print "SUBMITTING_CMD:\t\t$cmd\n";
+    foreach (1..$required_worker_count) {
+        print "SUBMITTING_CMD:\t\t$rc_specific_submission_cmd_args $cmd\n";
         system( $cmd );
     }
 }

modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm

@@ -137,17 +137,17 @@ sub find_out_causes {
 
 
 sub submit_workers {
-    my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_parameters, $submit_stdout_file, $submit_stderr_file) = @_;
+    my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_stdout_file, $submit_stderr_file) = @_;
 
-    my $job_name       = $self->generate_job_name($required_worker_count, $iteration, $rc_name);
-    my $submission_options = $self->config_get('SubmissionOptions');
+    my $job_name                            = $self->generate_job_name($required_worker_count, $iteration, $rc_name);
+    my $meadow_specific_submission_cmd_args = $self->config_get('SubmissionOptions');
 
-    $submit_stdout_file ||= '/dev/null';
-    $submit_stderr_file ||= '/dev/null';
+    $submit_stdout_file ||= '/dev/null';    # a value is required
+    $submit_stderr_file ||= '/dev/null';    # a value is required
 
     $ENV{'LSB_STDOUT_DIRECT'} = 'y';  # unbuffer the output of the bsub command
 
-    my $cmd = qq{bsub -o $submit_stdout_file -e $submit_stderr_file -J "${job_name}" $rc_parameters $submission_options $worker_cmd};
+    my $cmd = qq{bsub -o $submit_stdout_file -e $submit_stderr_file -J "${job_name}" $rc_specific_submission_cmd_args $meadow_specific_submission_cmd_args $worker_cmd};
 
     print "SUBMITTING_CMD:\t\t$cmd\n";
     system($cmd) && die "Could not submit job(s): $!, $?";  # let's abort the beekeeper and let the user check the syntax

modules/Bio/EnsEMBL/Hive/PipeConfig/HiveGeneric_conf.pm

@@ -494,11 +494,14 @@ sub run {
                 warn "Attempt to re-create and potentially redefine resource_class $rc_name($rc_id). NB: This may affect already created analyses!\n";
             }
 
-            while( my($meadow_type, $xparams) = each %$mt2param ) {
+            while( my($meadow_type, $resource_param_list) = each %$mt2param ) {
+                $resource_param_list = [ $resource_param_list ] unless(ref($resource_param_list));  # expecting either a scalar or a 2-element array
+
                 $resource_description_adaptor->create_new(
-                    -RESOURCE_CLASS_ID  => $rc_id,
-                    -MEADOW_TYPE        => $meadow_type,
-                    -PARAMETERS         => $xparams,
+                    -resource_class_id      => $rc_id,
+                    -meadow_type            => $meadow_type,
+                    -submission_cmd_args    => $resource_param_list->[0],
+                    -worker_cmd_args        => $resource_param_list->[1],
                 );
             }
         }

modules/Bio/EnsEMBL/Hive/ResourceDescription.pm

@@ -13,7 +13,9 @@
     CREATE TABLE resource_description (
         resource_class_id     int(10) unsigned NOT NULL,
         meadow_type           varchar(40) NOT NULL,
-        parameters            varchar(255) DEFAULT '' NOT NULL,
+        submission_cmd_args     VARCHAR(255) NOT NULL DEFAULT '',
+        worker_cmd_args         VARCHAR(255) NOT NULL DEFAULT '',
+
         PRIMARY KEY(resource_class_id, meadow_type)
     ) ENGINE=InnoDB;
 
@@ -35,13 +37,14 @@ sub new {
 
     my $self = bless {}, $class;
 
-    my ($adaptor, $resource_class_id, $meadow_type, $parameters) =
-         rearrange([qw(adaptor resource_class_id meadow_type parameters) ], @_);
+    my ($adaptor, $resource_class_id, $meadow_type, $submission_cmd_args, $worker_cmd_args) =
+         rearrange([qw(adaptor resource_class_id meadow_type submission_cmd_args worker_cmd_args) ], @_);
 
     $self->adaptor($adaptor) if(defined($adaptor));
     $self->resource_class_id($resource_class_id);
     $self->meadow_type($meadow_type);
-    $self->parameters($parameters);
+    $self->submission_cmd_args($submission_cmd_args);
+    $self->worker_cmd_args($worker_cmd_args);
 
     return $self;
 }
@@ -68,6 +71,7 @@ sub resource_class_id {
     return $self->{'_resource_class_id'};
 }
 
+
 sub meadow_type {
     my $self = shift @_;
 
@@ -77,19 +81,31 @@ sub meadow_type {
     return $self->{'_meadow_type'};
 }
 
-sub parameters {
+
+sub submission_cmd_args {
     my $self = shift @_;
 
     if(@_) {
-        $self->{'_parameters'} = shift @_;
+        $self->{'_submission_cmd_args'} = shift @_;
     }
-    return $self->{'_parameters'};
+    return $self->{'_submission_cmd_args'} || '';
 }
 
+
+sub worker_cmd_args {
+    my $self = shift @_;
+
+    if(@_) {
+        $self->{'_worker_cmd_args'} = shift @_;
+    }
+    return $self->{'_worker_cmd_args'} || '';
+}
+
+
 sub toString {
     my $self = shift @_;
 
-    return (ref($self).': '.join(', ', map { $_.'="'.$self->$_().'"' } qw(resource_class_id meadow_type parameters) ));
+    return (ref($self).': '.join(', ', map { $_.'="'.$self->$_().'"' } qw(resource_class_id meadow_type submission_cmd_args worker_cmd_args) ));
 }
 
 1;

scripts/beekeeper.pl

@@ -323,13 +323,13 @@ sub generate_worker_cmd {
 sub run_autonomously {
     my ($self, $max_loops, $keep_alive, $queen, $valley, $run_analysis, $run_job_id, $force) = @_;
 
-    my $worker_cmd      = generate_worker_cmd($self, $run_analysis, $run_job_id, $force);
-    my $special_task    = $run_analysis || $run_job_id;
+    my $resourceless_worker_cmd = generate_worker_cmd($self, $run_analysis, $run_job_id, $force);
+    my $special_task            = $run_analysis || $run_job_id;
 
     my $rc_id2name  = $self->{'dba'}->get_ResourceClassAdaptor->fetch_HASHED_FROM_resource_class_id_TO_name();
-    my %meadow_type_rc_name2xparams = ();
+    my %meadow_type_rc_name2resource_param_list = ();
     foreach my $rd (@{ $self->{'dba'}->get_ResourceDescriptionAdaptor->fetch_all() }) {
-        $meadow_type_rc_name2xparams{ $rd->meadow_type() }{ $rc_id2name->{$rd->resource_class_id} } = $rd->parameters();
+        $meadow_type_rc_name2resource_param_list{ $rd->meadow_type() }{ $rc_id2name->{$rd->resource_class_id} } = [ $rd->submission_cmd_args, $rd->worker_cmd_args ];
     }
 
     my $iteration=0;
@@ -363,8 +363,14 @@ sub run_autonomously {
 
                     print "Submitting $this_meadow_rc_worker_count workers (rc_name=$rc_name) to ".$this_meadow->signature()."\n";
 
-                    $this_meadow->submit_workers($worker_cmd.($special_task ? '' : " -rc_name $rc_name"), $this_meadow_rc_worker_count, $iteration,
-                                                    $rc_name, $meadow_type_rc_name2xparams{ $meadow_type }{ $rc_name } || '',
+                    my ($submission_cmd_args, $worker_cmd_args) = @{ $meadow_type_rc_name2resource_param_list{ $meadow_type }{ $rc_name } || [] };
+
+                    my $specific_worker_cmd = $resourceless_worker_cmd
+                                            . ($special_task ? '' : " -rc_name $rc_name")
+                                            . (defined($worker_cmd_args) ? " $worker_cmd_args" : '');
+
+                    $this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration,
+                                                    $rc_name, $submission_cmd_args || '',
                                                     $self->{'submit_stdout_file'}, $self->{'submit_stderr_file'});
                 }
             }

sql/patch_2013-11-12.mysql

+
+    -- Rename resource_description.parameters to stop confusion with job/analysis/pipeline parameters:
+ALTER TABLE resource_description CHANGE COLUMN parameters submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
+
+    -- Add resource-specific worker_cmd_args :
+ALTER TABLE resource_description ADD COLUMN worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
+
+    -- UPDATE hive_sql_schema_version
+UPDATE hive_meta SET meta_value=54 WHERE meta_key='hive_sql_schema_version' AND meta_value='53';
+

sql/patch_2013-11-12.pgsql

+
+    -- Rename resource_description.parameters to stop confusion with job/analysis/pipeline parameters:
+ALTER TABLE resource_description RENAME COLUMN parameters TO submission_cmd_args;
+
+    -- Add resource-specific worker_cmd_args :
+ALTER TABLE resource_description ADD COLUMN worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
+
+    -- UPDATE hive_sql_schema_version
+UPDATE hive_meta SET meta_value=54 WHERE meta_key='hive_sql_schema_version' AND meta_value='53';
+

sql/tables.mysql

@@ -217,13 +217,15 @@ CREATE TABLE resource_class (
 
 @column resource_class_id   foreign-keyed to the ResourceClass entry
 @column meadow_type         if the Worker is about to be executed on the given Meadow...
-@column parameters          ... the following resource line should be given to it.
+@column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
+@column worker_cmd_args     ... and these are the arguments that are given to the worker command being submitted
 */
 
 CREATE TABLE resource_description (
     resource_class_id       INTEGER      NOT NULL,
     meadow_type             VARCHAR(255) NOT NULL,
-    parameters              VARCHAR(255) NOT NULL DEFAULT '',
+    submission_cmd_args     VARCHAR(255) NOT NULL DEFAULT '',
+    worker_cmd_args         VARCHAR(255) NOT NULL DEFAULT '',
 
     PRIMARY KEY(resource_class_id, meadow_type)
 ) COLLATE=latin1_swedish_ci ENGINE=InnoDB;

sql/tables.pgsql

@@ -221,12 +221,15 @@ CREATE TABLE resource_class (
 @column resource_class_id   foreign-keyed to the ResourceClass entry
 @column meadow_type         if the Worker is about to be executed on the given Meadow...
 @column parameters          ... the following resource line should be given to it.
+@column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
+@column worker_cmd_args     ... and these are the arguments that are given to the worker command being submitted
 */
 
 CREATE TABLE resource_description (
     resource_class_id       INTEGER     NOT NULL,
     meadow_type             VARCHAR(255) NOT NULL,
-    parameters              VARCHAR(255) NOT NULL DEFAULT '',
+    submission_cmd_args     VARCHAR(255) NOT NULL DEFAULT '',
+    worker_cmd_args         VARCHAR(255) NOT NULL DEFAULT '',
 
     PRIMARY KEY(resource_class_id, meadow_type)
 );

sql/tables.sqlite

@@ -214,13 +214,15 @@ CREATE UNIQUE INDEX resource_class_name_idx ON resource_class (name);
 
 @column resource_class_id   foreign-keyed to the ResourceClass entry
 @column meadow_type         if the Worker is about to be executed on the given Meadow...
-@column parameters          ... the following resource line should be given to it.
+@column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
+@column worker_cmd_args     ... and these are the arguments that are given to the worker command being submitted
 */
 
 CREATE TABLE resource_description (
     resource_class_id       INTEGER     NOT NULL,
     meadow_type             VARCHAR(255) NOT NULL,
-    parameters              VARCHAR(255) NOT NULL DEFAULT '',
+    submission_cmd_args     VARCHAR(255) NOT NULL DEFAULT '',
+    worker_cmd_args         VARCHAR(255) NOT NULL DEFAULT '',
 
     PRIMARY KEY(resource_class_id, meadow_type)
 );