Commit f0be65dd authored by Leo Gordon's avatar Leo Gordon
Browse files

ResourceDescription expanded to include both submission_cmd_args and...

ResourceDescription expanded to include both submission_cmd_args and worker_cmd_args. Both args can be specified in a PipeConfig file.
parent 51e1dbb8
...@@ -94,12 +94,12 @@ sub kill_worker { ...@@ -94,12 +94,12 @@ sub kill_worker {
sub submit_workers { sub submit_workers {
my ($self, $worker_cmd, $worker_count, $iteration, $rc_name, $rc_parameters) = @_; my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_stdout_file, $submit_stderr_file) = @_;
my $cmd = "$worker_cmd &"; my $cmd = "$worker_cmd &";
foreach (1..$worker_count) { foreach (1..$required_worker_count) {
print "SUBMITTING_CMD:\t\t$cmd\n"; print "SUBMITTING_CMD:\t\t$rc_specific_submission_cmd_args $cmd\n";
system( $cmd ); system( $cmd );
} }
} }
......
...@@ -137,17 +137,17 @@ sub find_out_causes { ...@@ -137,17 +137,17 @@ sub find_out_causes {
sub submit_workers { sub submit_workers {
my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_parameters, $submit_stdout_file, $submit_stderr_file) = @_; my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_stdout_file, $submit_stderr_file) = @_;
my $job_name = $self->generate_job_name($required_worker_count, $iteration, $rc_name); my $job_name = $self->generate_job_name($required_worker_count, $iteration, $rc_name);
my $submission_options = $self->config_get('SubmissionOptions'); my $meadow_specific_submission_cmd_args = $self->config_get('SubmissionOptions');
$submit_stdout_file ||= '/dev/null'; $submit_stdout_file ||= '/dev/null'; # a value is required
$submit_stderr_file ||= '/dev/null'; $submit_stderr_file ||= '/dev/null'; # a value is required
$ENV{'LSB_STDOUT_DIRECT'} = 'y'; # unbuffer the output of the bsub command $ENV{'LSB_STDOUT_DIRECT'} = 'y'; # unbuffer the output of the bsub command
my $cmd = qq{bsub -o $submit_stdout_file -e $submit_stderr_file -J "${job_name}" $rc_parameters $submission_options $worker_cmd}; my $cmd = qq{bsub -o $submit_stdout_file -e $submit_stderr_file -J "${job_name}" $rc_specific_submission_cmd_args $meadow_specific_submission_cmd_args $worker_cmd};
print "SUBMITTING_CMD:\t\t$cmd\n"; print "SUBMITTING_CMD:\t\t$cmd\n";
system($cmd) && die "Could not submit job(s): $!, $?"; # let's abort the beekeeper and let the user check the syntax system($cmd) && die "Could not submit job(s): $!, $?"; # let's abort the beekeeper and let the user check the syntax
......
...@@ -494,11 +494,14 @@ sub run { ...@@ -494,11 +494,14 @@ sub run {
warn "Attempt to re-create and potentially redefine resource_class $rc_name($rc_id). NB: This may affect already created analyses!\n"; warn "Attempt to re-create and potentially redefine resource_class $rc_name($rc_id). NB: This may affect already created analyses!\n";
} }
while( my($meadow_type, $xparams) = each %$mt2param ) { while( my($meadow_type, $resource_param_list) = each %$mt2param ) {
$resource_param_list = [ $resource_param_list ] unless(ref($resource_param_list)); # expecting either a scalar or a 2-element array
$resource_description_adaptor->create_new( $resource_description_adaptor->create_new(
-RESOURCE_CLASS_ID => $rc_id, -resource_class_id => $rc_id,
-MEADOW_TYPE => $meadow_type, -meadow_type => $meadow_type,
-PARAMETERS => $xparams, -submission_cmd_args => $resource_param_list->[0],
-worker_cmd_args => $resource_param_list->[1],
); );
} }
} }
......
...@@ -13,7 +13,9 @@ ...@@ -13,7 +13,9 @@
CREATE TABLE resource_description ( CREATE TABLE resource_description (
resource_class_id int(10) unsigned NOT NULL, resource_class_id int(10) unsigned NOT NULL,
meadow_type varchar(40) NOT NULL, meadow_type varchar(40) NOT NULL,
parameters varchar(255) DEFAULT '' NOT NULL, submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY(resource_class_id, meadow_type) PRIMARY KEY(resource_class_id, meadow_type)
) ENGINE=InnoDB; ) ENGINE=InnoDB;
...@@ -35,13 +37,14 @@ sub new { ...@@ -35,13 +37,14 @@ sub new {
my $self = bless {}, $class; my $self = bless {}, $class;
my ($adaptor, $resource_class_id, $meadow_type, $parameters) = my ($adaptor, $resource_class_id, $meadow_type, $submission_cmd_args, $worker_cmd_args) =
rearrange([qw(adaptor resource_class_id meadow_type parameters) ], @_); rearrange([qw(adaptor resource_class_id meadow_type submission_cmd_args worker_cmd_args) ], @_);
$self->adaptor($adaptor) if(defined($adaptor)); $self->adaptor($adaptor) if(defined($adaptor));
$self->resource_class_id($resource_class_id); $self->resource_class_id($resource_class_id);
$self->meadow_type($meadow_type); $self->meadow_type($meadow_type);
$self->parameters($parameters); $self->submission_cmd_args($submission_cmd_args);
$self->worker_cmd_args($worker_cmd_args);
return $self; return $self;
} }
...@@ -68,6 +71,7 @@ sub resource_class_id { ...@@ -68,6 +71,7 @@ sub resource_class_id {
return $self->{'_resource_class_id'}; return $self->{'_resource_class_id'};
} }
sub meadow_type { sub meadow_type {
my $self = shift @_; my $self = shift @_;
...@@ -77,19 +81,31 @@ sub meadow_type { ...@@ -77,19 +81,31 @@ sub meadow_type {
return $self->{'_meadow_type'}; return $self->{'_meadow_type'};
} }
sub parameters {
sub submission_cmd_args {
my $self = shift @_; my $self = shift @_;
if(@_) { if(@_) {
$self->{'_parameters'} = shift @_; $self->{'_submission_cmd_args'} = shift @_;
} }
return $self->{'_parameters'}; return $self->{'_submission_cmd_args'} || '';
} }
sub worker_cmd_args {
my $self = shift @_;
if(@_) {
$self->{'_worker_cmd_args'} = shift @_;
}
return $self->{'_worker_cmd_args'} || '';
}
sub toString { sub toString {
my $self = shift @_; my $self = shift @_;
return (ref($self).': '.join(', ', map { $_.'="'.$self->$_().'"' } qw(resource_class_id meadow_type parameters) )); return (ref($self).': '.join(', ', map { $_.'="'.$self->$_().'"' } qw(resource_class_id meadow_type submission_cmd_args worker_cmd_args) ));
} }
1; 1;
......
...@@ -323,13 +323,13 @@ sub generate_worker_cmd { ...@@ -323,13 +323,13 @@ sub generate_worker_cmd {
sub run_autonomously { sub run_autonomously {
my ($self, $max_loops, $keep_alive, $queen, $valley, $run_analysis, $run_job_id, $force) = @_; my ($self, $max_loops, $keep_alive, $queen, $valley, $run_analysis, $run_job_id, $force) = @_;
my $worker_cmd = generate_worker_cmd($self, $run_analysis, $run_job_id, $force); my $resourceless_worker_cmd = generate_worker_cmd($self, $run_analysis, $run_job_id, $force);
my $special_task = $run_analysis || $run_job_id; my $special_task = $run_analysis || $run_job_id;
my $rc_id2name = $self->{'dba'}->get_ResourceClassAdaptor->fetch_HASHED_FROM_resource_class_id_TO_name(); my $rc_id2name = $self->{'dba'}->get_ResourceClassAdaptor->fetch_HASHED_FROM_resource_class_id_TO_name();
my %meadow_type_rc_name2xparams = (); my %meadow_type_rc_name2resource_param_list = ();
foreach my $rd (@{ $self->{'dba'}->get_ResourceDescriptionAdaptor->fetch_all() }) { foreach my $rd (@{ $self->{'dba'}->get_ResourceDescriptionAdaptor->fetch_all() }) {
$meadow_type_rc_name2xparams{ $rd->meadow_type() }{ $rc_id2name->{$rd->resource_class_id} } = $rd->parameters(); $meadow_type_rc_name2resource_param_list{ $rd->meadow_type() }{ $rc_id2name->{$rd->resource_class_id} } = [ $rd->submission_cmd_args, $rd->worker_cmd_args ];
} }
my $iteration=0; my $iteration=0;
...@@ -363,8 +363,14 @@ sub run_autonomously { ...@@ -363,8 +363,14 @@ sub run_autonomously {
print "Submitting $this_meadow_rc_worker_count workers (rc_name=$rc_name) to ".$this_meadow->signature()."\n"; print "Submitting $this_meadow_rc_worker_count workers (rc_name=$rc_name) to ".$this_meadow->signature()."\n";
$this_meadow->submit_workers($worker_cmd.($special_task ? '' : " -rc_name $rc_name"), $this_meadow_rc_worker_count, $iteration, my ($submission_cmd_args, $worker_cmd_args) = @{ $meadow_type_rc_name2resource_param_list{ $meadow_type }{ $rc_name } || [] };
$rc_name, $meadow_type_rc_name2xparams{ $meadow_type }{ $rc_name } || '',
my $specific_worker_cmd = $resourceless_worker_cmd
. ($special_task ? '' : " -rc_name $rc_name")
. (defined($worker_cmd_args) ? " $worker_cmd_args" : '');
$this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration,
$rc_name, $submission_cmd_args || '',
$self->{'submit_stdout_file'}, $self->{'submit_stderr_file'}); $self->{'submit_stdout_file'}, $self->{'submit_stderr_file'});
} }
} }
......
-- Rename resource_description.parameters to stop confusion with job/analysis/pipeline parameters:
ALTER TABLE resource_description CHANGE COLUMN parameters submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
-- Add resource-specific worker_cmd_args :
ALTER TABLE resource_description ADD COLUMN worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
-- UPDATE hive_sql_schema_version
UPDATE hive_meta SET meta_value=54 WHERE meta_key='hive_sql_schema_version' AND meta_value='53';
-- Rename resource_description.parameters to stop confusion with job/analysis/pipeline parameters:
ALTER TABLE resource_description RENAME COLUMN parameters TO submission_cmd_args;
-- Add resource-specific worker_cmd_args :
ALTER TABLE resource_description ADD COLUMN worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
-- UPDATE hive_sql_schema_version
UPDATE hive_meta SET meta_value=54 WHERE meta_key='hive_sql_schema_version' AND meta_value='53';
...@@ -217,13 +217,15 @@ CREATE TABLE resource_class ( ...@@ -217,13 +217,15 @@ CREATE TABLE resource_class (
@column resource_class_id foreign-keyed to the ResourceClass entry @column resource_class_id foreign-keyed to the ResourceClass entry
@column meadow_type if the Worker is about to be executed on the given Meadow... @column meadow_type if the Worker is about to be executed on the given Meadow...
@column parameters ... the following resource line should be given to it. @column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
@column worker_cmd_args ... and these are the arguments that are given to the worker command being submitted
*/ */
CREATE TABLE resource_description ( CREATE TABLE resource_description (
resource_class_id INTEGER NOT NULL, resource_class_id INTEGER NOT NULL,
meadow_type VARCHAR(255) NOT NULL, meadow_type VARCHAR(255) NOT NULL,
parameters VARCHAR(255) NOT NULL DEFAULT '', submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY(resource_class_id, meadow_type) PRIMARY KEY(resource_class_id, meadow_type)
) COLLATE=latin1_swedish_ci ENGINE=InnoDB; ) COLLATE=latin1_swedish_ci ENGINE=InnoDB;
......
...@@ -221,12 +221,15 @@ CREATE TABLE resource_class ( ...@@ -221,12 +221,15 @@ CREATE TABLE resource_class (
@column resource_class_id foreign-keyed to the ResourceClass entry @column resource_class_id foreign-keyed to the ResourceClass entry
@column meadow_type if the Worker is about to be executed on the given Meadow... @column meadow_type if the Worker is about to be executed on the given Meadow...
@column parameters ... the following resource line should be given to it. @column parameters ... the following resource line should be given to it.
@column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
@column worker_cmd_args ... and these are the arguments that are given to the worker command being submitted
*/ */
CREATE TABLE resource_description ( CREATE TABLE resource_description (
resource_class_id INTEGER NOT NULL, resource_class_id INTEGER NOT NULL,
meadow_type VARCHAR(255) NOT NULL, meadow_type VARCHAR(255) NOT NULL,
parameters VARCHAR(255) NOT NULL DEFAULT '', submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY(resource_class_id, meadow_type) PRIMARY KEY(resource_class_id, meadow_type)
); );
......
...@@ -214,13 +214,15 @@ CREATE UNIQUE INDEX resource_class_name_idx ON resource_class (name); ...@@ -214,13 +214,15 @@ CREATE UNIQUE INDEX resource_class_name_idx ON resource_class (name);
@column resource_class_id foreign-keyed to the ResourceClass entry @column resource_class_id foreign-keyed to the ResourceClass entry
@column meadow_type if the Worker is about to be executed on the given Meadow... @column meadow_type if the Worker is about to be executed on the given Meadow...
@column parameters ... the following resource line should be given to it. @column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
@column worker_cmd_args ... and these are the arguments that are given to the worker command being submitted
*/ */
CREATE TABLE resource_description ( CREATE TABLE resource_description (
resource_class_id INTEGER NOT NULL, resource_class_id INTEGER NOT NULL,
meadow_type VARCHAR(255) NOT NULL, meadow_type VARCHAR(255) NOT NULL,
parameters VARCHAR(255) NOT NULL DEFAULT '', submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY(resource_class_id, meadow_type) PRIMARY KEY(resource_class_id, meadow_type)
); );
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment