Commit f0be65dd authored by Leo Gordon's avatar Leo Gordon
Browse files

ResourceDescription expanded to include both submission_cmd_args and...

ResourceDescription expanded to include both submission_cmd_args and worker_cmd_args. Both args can be specified in a PipeConfig file.
parent 51e1dbb8
......@@ -94,12 +94,12 @@ sub kill_worker {
sub submit_workers {
my ($self, $worker_cmd, $worker_count, $iteration, $rc_name, $rc_parameters) = @_;
my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_stdout_file, $submit_stderr_file) = @_;
my $cmd = "$worker_cmd &";
foreach (1..$worker_count) {
print "SUBMITTING_CMD:\t\t$cmd\n";
foreach (1..$required_worker_count) {
print "SUBMITTING_CMD:\t\t$rc_specific_submission_cmd_args $cmd\n";
system( $cmd );
}
}
......
......@@ -137,17 +137,17 @@ sub find_out_causes {
sub submit_workers {
my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_parameters, $submit_stdout_file, $submit_stderr_file) = @_;
my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_stdout_file, $submit_stderr_file) = @_;
my $job_name = $self->generate_job_name($required_worker_count, $iteration, $rc_name);
my $submission_options = $self->config_get('SubmissionOptions');
my $job_name = $self->generate_job_name($required_worker_count, $iteration, $rc_name);
my $meadow_specific_submission_cmd_args = $self->config_get('SubmissionOptions');
$submit_stdout_file ||= '/dev/null';
$submit_stderr_file ||= '/dev/null';
$submit_stdout_file ||= '/dev/null'; # a value is required
$submit_stderr_file ||= '/dev/null'; # a value is required
$ENV{'LSB_STDOUT_DIRECT'} = 'y'; # unbuffer the output of the bsub command
my $cmd = qq{bsub -o $submit_stdout_file -e $submit_stderr_file -J "${job_name}" $rc_parameters $submission_options $worker_cmd};
my $cmd = qq{bsub -o $submit_stdout_file -e $submit_stderr_file -J "${job_name}" $rc_specific_submission_cmd_args $meadow_specific_submission_cmd_args $worker_cmd};
print "SUBMITTING_CMD:\t\t$cmd\n";
system($cmd) && die "Could not submit job(s): $!, $?"; # let's abort the beekeeper and let the user check the syntax
......
......@@ -494,11 +494,14 @@ sub run {
warn "Attempt to re-create and potentially redefine resource_class $rc_name($rc_id). NB: This may affect already created analyses!\n";
}
while( my($meadow_type, $xparams) = each %$mt2param ) {
while( my($meadow_type, $resource_param_list) = each %$mt2param ) {
$resource_param_list = [ $resource_param_list ] unless(ref($resource_param_list)); # expecting either a scalar or a 2-element array
$resource_description_adaptor->create_new(
-RESOURCE_CLASS_ID => $rc_id,
-MEADOW_TYPE => $meadow_type,
-PARAMETERS => $xparams,
-resource_class_id => $rc_id,
-meadow_type => $meadow_type,
-submission_cmd_args => $resource_param_list->[0],
-worker_cmd_args => $resource_param_list->[1],
);
}
}
......
......@@ -13,7 +13,9 @@
CREATE TABLE resource_description (
resource_class_id int(10) unsigned NOT NULL,
meadow_type varchar(40) NOT NULL,
parameters varchar(255) DEFAULT '' NOT NULL,
submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY(resource_class_id, meadow_type)
) ENGINE=InnoDB;
......@@ -35,13 +37,14 @@ sub new {
my $self = bless {}, $class;
my ($adaptor, $resource_class_id, $meadow_type, $parameters) =
rearrange([qw(adaptor resource_class_id meadow_type parameters) ], @_);
my ($adaptor, $resource_class_id, $meadow_type, $submission_cmd_args, $worker_cmd_args) =
rearrange([qw(adaptor resource_class_id meadow_type submission_cmd_args worker_cmd_args) ], @_);
$self->adaptor($adaptor) if(defined($adaptor));
$self->resource_class_id($resource_class_id);
$self->meadow_type($meadow_type);
$self->parameters($parameters);
$self->submission_cmd_args($submission_cmd_args);
$self->worker_cmd_args($worker_cmd_args);
return $self;
}
......@@ -68,6 +71,7 @@ sub resource_class_id {
return $self->{'_resource_class_id'};
}
sub meadow_type {
my $self = shift @_;
......@@ -77,19 +81,31 @@ sub meadow_type {
return $self->{'_meadow_type'};
}
sub parameters {
sub submission_cmd_args {
my $self = shift @_;
if(@_) {
$self->{'_parameters'} = shift @_;
$self->{'_submission_cmd_args'} = shift @_;
}
return $self->{'_parameters'};
return $self->{'_submission_cmd_args'} || '';
}
sub worker_cmd_args {
my $self = shift @_;
if(@_) {
$self->{'_worker_cmd_args'} = shift @_;
}
return $self->{'_worker_cmd_args'} || '';
}
sub toString {
my $self = shift @_;
return (ref($self).': '.join(', ', map { $_.'="'.$self->$_().'"' } qw(resource_class_id meadow_type parameters) ));
return (ref($self).': '.join(', ', map { $_.'="'.$self->$_().'"' } qw(resource_class_id meadow_type submission_cmd_args worker_cmd_args) ));
}
1;
......
......@@ -323,13 +323,13 @@ sub generate_worker_cmd {
sub run_autonomously {
my ($self, $max_loops, $keep_alive, $queen, $valley, $run_analysis, $run_job_id, $force) = @_;
my $worker_cmd = generate_worker_cmd($self, $run_analysis, $run_job_id, $force);
my $special_task = $run_analysis || $run_job_id;
my $resourceless_worker_cmd = generate_worker_cmd($self, $run_analysis, $run_job_id, $force);
my $special_task = $run_analysis || $run_job_id;
my $rc_id2name = $self->{'dba'}->get_ResourceClassAdaptor->fetch_HASHED_FROM_resource_class_id_TO_name();
my %meadow_type_rc_name2xparams = ();
my %meadow_type_rc_name2resource_param_list = ();
foreach my $rd (@{ $self->{'dba'}->get_ResourceDescriptionAdaptor->fetch_all() }) {
$meadow_type_rc_name2xparams{ $rd->meadow_type() }{ $rc_id2name->{$rd->resource_class_id} } = $rd->parameters();
$meadow_type_rc_name2resource_param_list{ $rd->meadow_type() }{ $rc_id2name->{$rd->resource_class_id} } = [ $rd->submission_cmd_args, $rd->worker_cmd_args ];
}
my $iteration=0;
......@@ -363,8 +363,14 @@ sub run_autonomously {
print "Submitting $this_meadow_rc_worker_count workers (rc_name=$rc_name) to ".$this_meadow->signature()."\n";
$this_meadow->submit_workers($worker_cmd.($special_task ? '' : " -rc_name $rc_name"), $this_meadow_rc_worker_count, $iteration,
$rc_name, $meadow_type_rc_name2xparams{ $meadow_type }{ $rc_name } || '',
my ($submission_cmd_args, $worker_cmd_args) = @{ $meadow_type_rc_name2resource_param_list{ $meadow_type }{ $rc_name } || [] };
my $specific_worker_cmd = $resourceless_worker_cmd
. ($special_task ? '' : " -rc_name $rc_name")
. (defined($worker_cmd_args) ? " $worker_cmd_args" : '');
$this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration,
$rc_name, $submission_cmd_args || '',
$self->{'submit_stdout_file'}, $self->{'submit_stderr_file'});
}
}
......
-- Rename resource_description.parameters to stop confusion with job/analysis/pipeline parameters:
ALTER TABLE resource_description CHANGE COLUMN parameters submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
-- Add resource-specific worker_cmd_args :
ALTER TABLE resource_description ADD COLUMN worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
-- UPDATE hive_sql_schema_version
UPDATE hive_meta SET meta_value=54 WHERE meta_key='hive_sql_schema_version' AND meta_value='53';
-- Rename resource_description.parameters to stop confusion with job/analysis/pipeline parameters:
ALTER TABLE resource_description RENAME COLUMN parameters TO submission_cmd_args;
-- Add resource-specific worker_cmd_args :
ALTER TABLE resource_description ADD COLUMN worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '';
-- UPDATE hive_sql_schema_version
UPDATE hive_meta SET meta_value=54 WHERE meta_key='hive_sql_schema_version' AND meta_value='53';
......@@ -217,13 +217,15 @@ CREATE TABLE resource_class (
@column resource_class_id foreign-keyed to the ResourceClass entry
@column meadow_type if the Worker is about to be executed on the given Meadow...
@column parameters ... the following resource line should be given to it.
@column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
@column worker_cmd_args ... and these are the arguments that are given to the worker command being submitted
*/
CREATE TABLE resource_description (
resource_class_id INTEGER NOT NULL,
meadow_type VARCHAR(255) NOT NULL,
parameters VARCHAR(255) NOT NULL DEFAULT '',
submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY(resource_class_id, meadow_type)
) COLLATE=latin1_swedish_ci ENGINE=InnoDB;
......
......@@ -221,12 +221,15 @@ CREATE TABLE resource_class (
@column resource_class_id foreign-keyed to the ResourceClass entry
@column meadow_type if the Worker is about to be executed on the given Meadow...
@column parameters ... the following resource line should be given to it.
@column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
@column worker_cmd_args ... and these are the arguments that are given to the worker command being submitted
*/
CREATE TABLE resource_description (
resource_class_id INTEGER NOT NULL,
meadow_type VARCHAR(255) NOT NULL,
parameters VARCHAR(255) NOT NULL DEFAULT '',
submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY(resource_class_id, meadow_type)
);
......
......@@ -214,13 +214,15 @@ CREATE UNIQUE INDEX resource_class_name_idx ON resource_class (name);
@column resource_class_id foreign-keyed to the ResourceClass entry
@column meadow_type if the Worker is about to be executed on the given Meadow...
@column parameters ... the following resource line should be given to it.
@column submission_cmd_args ... these are the resource arguments (queue, memory,...) to give to the submission command
@column worker_cmd_args ... and these are the arguments that are given to the worker command being submitted
*/
CREATE TABLE resource_description (
resource_class_id INTEGER NOT NULL,
meadow_type VARCHAR(255) NOT NULL,
parameters VARCHAR(255) NOT NULL DEFAULT '',
submission_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
worker_cmd_args VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY(resource_class_id, meadow_type)
);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment