Commit f00c64c5 authored by Leo Gordon's avatar Leo Gordon
Browse files

schema_change: preregister Workers upon submission in SUBMITTED state, added...

schema_change: preregister Workers upon submission in SUBMITTED state, added when_submitted, relaxed cause_of_death to VARCHAR
parent 08145c68
......@@ -117,7 +117,7 @@ sub kill_worker {
}
sub submit_workers {
sub submit_workers_return_meadow_pids {
my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_log_subdir) = @_;
my @worker_cmd_components = split_for_bash($worker_cmd); # FIXME: change the interface so that $worker_cmd itself is passed in as ARRAYref
......
......@@ -299,7 +299,7 @@ sub get_report_entries_for_time_interval {
}
sub submit_workers {
sub submit_workers_return_meadow_pids {
my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_specific_submission_cmd_args, $submit_log_subdir) = @_;
my $job_array_common_name = $self->job_array_common_name($rc_name, $iteration);
......
......@@ -112,9 +112,9 @@ sub create_new_worker {
my $self = shift @_;
my %flags = @_;
my ($resource_class_id, $resource_class_name, $beekeeper_id,
my ($preregistered, $resource_class_id, $resource_class_name, $beekeeper_id,
$no_write, $debug, $worker_log_dir, $hive_log_dir, $job_limit, $life_span, $no_cleanup, $retry_throwing_jobs, $can_respecialize)
= @flags{qw(-resource_class_id -resource_class_name -beekeeper_id
= @flags{qw(-preregistered -resource_class_id -resource_class_name -beekeeper_id
-no_write -debug -worker_log_dir -hive_log_dir -job_limit -life_span -no_cleanup -retry_throwing_jobs -can_respecialize)};
my ($meadow, $process_id, $meadow_host, $meadow_user) = Bio::EnsEMBL::Hive::Valley->new()->whereami();
......@@ -123,7 +123,7 @@ sub create_new_worker {
my $meadow_name = $meadow->cached_name;
foreach my $prev_worker_incarnation (@{ $self->find_previous_worker_incarnations($meadow_type, $meadow_name, $process_id) }) {
# so far 'RELOCATED events' has been detected on LSF 9.0 in response to sending signal #99 or #100
# So far 'RELOCATED events' has been detected on LSF 9.0 in response to sending signal #99 or #100
# Since I don't know how to avoid them, I am trying to register them when they happen.
# The following snippet buries the previous incarnation of the Worker before starting a new one.
#
......@@ -134,6 +134,26 @@ sub create_new_worker {
$self->register_worker_death( $prev_worker_incarnation );
}
my $worker;
if($preregistered) {
my $sec = 1;
until( $worker = $self->fetch_preregistered_worker($meadow_type, $meadow_name, $process_id) ) {
$self->db->get_LogMessageAdaptor->store_hive_message("Preregistered Worker $meadow_type/$meadow_name:$process_id waiting $sec more seconds to fetch itself...", 'WORKER_CAUTION' );
sleep($sec);
$sec *= 2;
}
# only update the fields that were not available at the time of submission:
$worker->meadow_host( $meadow_host );
$worker->meadow_user( $meadow_user );
$worker->when_born( 'CURRENT_TIMESTAMP' );
$worker->status( 'READY' );
$self->update( $worker );
} else {
my $resource_class;
if( defined($resource_class_name) ) {
......@@ -144,21 +164,25 @@ sub create_new_worker {
or die "resource_class with dbID='$resource_class_id' could not be fetched from the database";
}
my $worker = Bio::EnsEMBL::Hive::Worker->new(
$worker = Bio::EnsEMBL::Hive::Worker->new(
'meadow_type' => $meadow_type,
'meadow_name' => $meadow_name,
'meadow_host' => $meadow_host,
'meadow_user' => $meadow_user,
'process_id' => $process_id,
'resource_class' => $resource_class,
'beekeeper_id' => $beekeeper_id,
'meadow_host' => $meadow_host,
'meadow_user' => $meadow_user,
);
if (ref($self)) {
$self->store( $worker );
my $worker_id = $worker->dbID;
$worker = $self->fetch_by_dbID( $worker_id ) # refresh the object to get the fields initialized at SQL level (timestamps in this case)
or die "Could not fetch worker with dbID=$worker_id";
$worker->when_born( 'CURRENT_TIMESTAMP' );
$self->update_when_born( $worker );
$self->refresh( $worker );
}
}
$worker->set_log_directory_name($hive_log_dir, $worker_log_dir);
......@@ -475,7 +499,19 @@ sub find_previous_worker_incarnations {
# This happens in standalone mode, when there is no database
return [] unless ref($self);
return $self->fetch_all( "status!='DEAD' AND meadow_type='$meadow_type' AND meadow_name='$meadow_name' AND process_id='$process_id'" );
return $self->fetch_all( "status!='DEAD' AND status!='SUBMITTED' AND meadow_type='$meadow_type' AND meadow_name='$meadow_name' AND process_id='$process_id'" );
}
sub fetch_preregistered_worker {
my ($self, $meadow_type, $meadow_name, $process_id) = @_;
# This happens in standalone mode, when there is no database
return [] unless ref($self);
my ($worker) = @{ $self->fetch_all( "status='SUBMITTED' AND meadow_type='$meadow_type' AND meadow_name='$meadow_name' AND process_id='$process_id'" ) };
return $worker;
}
......
......@@ -52,6 +52,8 @@ sub runWorker {
# Create the worker
my $worker = $queen->create_new_worker(
-preregistered => $specialization_options->{'preregistered'},
# Resource class:
-resource_class_id => $specialization_options->{'resource_class_id'},
-resource_class_name => $specialization_options->{'resource_class_name'},
......
......@@ -166,12 +166,21 @@ sub status {
return $self->{'_status'};
}
sub beekeeper_id {
my $self = shift;
$self->{'_beekeeper_id'} = shift if(@_);
return $self->{'_beekeeper_id'} || undef;
}
sub when_submitted {
my $self = shift;
$self->{'_when_submitted'} = shift if(@_);
return $self->{'_when_submitted'};
}
sub when_born {
my $self = shift;
$self->{'_when_born'} = shift if(@_);
......
......@@ -426,7 +426,7 @@ sub generate_worker_cmd {
exit(1);
}
foreach my $worker_option ('url', 'reg_conf', 'reg_type', 'reg_alias', 'nosqlvc', 'beekeeper_id', 'job_limit', 'life_span', 'retry_throwing_jobs', 'can_respecialize', 'hive_log_dir', 'debug') {
foreach my $worker_option ('url', 'reg_conf', 'reg_type', 'reg_alias', 'nosqlvc', 'job_limit', 'life_span', 'retry_throwing_jobs', 'can_respecialize', 'hive_log_dir', 'debug') {
if(defined(my $value = $self->{$worker_option})) {
$worker_cmd .= " -${worker_option} $value";
}
......@@ -572,17 +572,42 @@ sub run_autonomously {
'INFO', 'ALIVE');
my ($submission_cmd_args, $worker_cmd_args) = @{ $meadow_type_rc_name2resource_param_list{ $meadow_type }{ $rc_name } || [] };
my $meadow_can_preregister_workers = UNIVERSAL::can($this_meadow, 'submit_workers_return_meadow_pids');
my $specific_worker_cmd = $resourceless_worker_cmd
. " -rc_name $rc_name"
. (defined($worker_cmd_args) ? " $worker_cmd_args" : '');
. (defined($worker_cmd_args) ? " $worker_cmd_args" : '')
. ( $meadow_can_preregister_workers
? ' -preregistered'
: ' -beekeeper_id '.$self->{'beekeeper_id'}." -rc_name $rc_name" );
# Since the newer version of Meadow interface can parse and return $meadow_process_ids, the Workers can be preregistered in the database:
if($meadow_can_preregister_workers) {
if( my $meadow_process_ids = $this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration,
$rc_name, $submission_cmd_args || '', $submit_log_subdir) ) {
# Since the newer version of Meadow interface can parse and return $meadow_process_ids, the Workers can be pre-registered in the database
my $meadow_process_ids = $this_meadow->submit_workers_return_meadow_pids(
$specific_worker_cmd, $this_meadow_rc_worker_count, $iteration, $rc_name, $submission_cmd_args || '', $submit_log_subdir);
warn "Submitted the following process_ids to ".$this_meadow->signature.": ".join(', ', @$meadow_process_ids)."\n";
my $resource_class = $pipeline->collection_of('ResourceClass')->find_one_by('name', $rc_name);
my $meadow_name = $this_meadow->cached_name;
my @pre_allocated_workers = map {
Bio::EnsEMBL::Hive::Worker->new(
'meadow_type' => $meadow_type, # non-unique key components
'meadow_name' => $meadow_name,
'process_id' => $_,
'resource_class' => $resource_class, # non-key, but known at the time of pre-allocation
'beekeeper_id' => $self->{'beekeeper_id'},
'status' => 'SUBMITTED',
)
} @$meadow_process_ids;
$queen->store( \@pre_allocated_workers );
} else {
$this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration, $rc_name, $submission_cmd_args || '', $submit_log_subdir);
}
}
}
......
......@@ -24,7 +24,7 @@ main();
sub main {
my ($url, $reg_conf, $reg_type, $reg_alias, $nosqlvc); # Connection parameters
my ($resource_class_id, $resource_class_name, $analyses_pattern, $analysis_id, $logic_name, $job_id, $force, $beekeeper_id); # Task specification parameters
my ($preregistered, $resource_class_id, $resource_class_name, $analyses_pattern, $analysis_id, $logic_name, $job_id, $force, $beekeeper_id); # Task specification parameters
my ($job_limit, $life_span, $no_cleanup, $no_write, $hive_log_dir, $worker_log_dir, $retry_throwing_jobs, $can_respecialize); # Worker control parameters
my ($help, $report_versions, $debug);
......@@ -38,6 +38,7 @@ sub main {
'nosqlvc=i' => \$nosqlvc, # can't use the binary "!" as it is a propagated option
# Task specification parameters:
'preregistered!' => \$preregistered,
'rc_id=i' => \$resource_class_id,
'rc_name=s' => \$resource_class_name,
'analyses_pattern=s' => \$analyses_pattern,
......@@ -116,6 +117,7 @@ sub main {
}
my %specialization_options = (
preregistered => $preregistered,
resource_class_id => $resource_class_id,
resource_class_name => $resource_class_name,
can_respecialize => $can_respecialize,
......
-- ---------------------------------------------------------------------------------------------------
SET @expected_version = 91;
-- make MySQL stop immediately after it encounters division by zero:
SET SESSION sql_mode='TRADITIONAL';
-- warn that we detected the schema version mismatch:
SELECT CONCAT( 'The patch only applies to schema version ',
@expected_version,
', but the current schema version is ',
meta_value,
', so skipping the rest.') AS ''
FROM hive_meta WHERE meta_key='hive_sql_schema_version' AND meta_value<>@expected_version;
-- cause division by zero only if current version differs from the expected one:
INSERT INTO hive_meta (meta_key, meta_value)
SELECT 'this_should_never_be_inserted', 1 FROM hive_meta WHERE NOT 1/(meta_key<>'hive_sql_schema_version' OR meta_value=@expected_version);
SELECT CONCAT( 'The patch seems to be compatible with schema version ',
@expected_version,
', applying the patch...') AS '';
-- Now undo the change so that we could patch potentially non-TRADITIONAL schema:
SET SESSION sql_mode='';
-- ----------------------------------<actual_patch> -------------------------------------------------
-- part_1: changes to introduce 'SUBMITTED' state of Workers:
ALTER TABLE worker MODIFY COLUMN meadow_host VARCHAR(255) DEFAULT NULL; -- allow it to be NULL for 'SUBMITTED' entries
ALTER TABLE worker MODIFY COLUMN when_born TIMESTAMP NULL;
ALTER TABLE worker ADD COLUMN when_submitted TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP AFTER beekeeper_id;
-- part_2: changes to allow extra meadow-specific exit statuses to be representable in the worker table:
ALTER TABLE worker MODIFY COLUMN cause_of_death VARCHAR(255) DEFAULT NULL;
-- ----------------------------------</actual_patch> -------------------------------------------------
-- increase the schema version by one and register the patch:
UPDATE hive_meta SET meta_value=meta_value+1 WHERE meta_key='hive_sql_schema_version';
INSERT INTO hive_meta (meta_key, meta_value) SELECT CONCAT("patched_to_", meta_value), CURRENT_TIMESTAMP FROM hive_meta WHERE meta_key = "hive_sql_schema_version";
-- ---------------------------------------------------------------------------------------------------
\set expected_version 91
\set ON_ERROR_STOP on
-- warn that we detected the schema version mismatch:
SELECT ('The patch only applies to schema version '
|| CAST(:expected_version AS VARCHAR)
|| ', but the current schema version is '
|| meta_value
|| ', so skipping the rest.') as incompatible_msg
FROM hive_meta WHERE meta_key='hive_sql_schema_version' AND meta_value!=CAST(:expected_version AS VARCHAR);
-- cause division by zero only if current version differs from the expected one:
INSERT INTO hive_meta (meta_key, meta_value)
SELECT 'this_should_never_be_inserted', 1 FROM hive_meta WHERE 1 != 1/CAST( (meta_key!='hive_sql_schema_version' OR meta_value=CAST(:expected_version AS VARCHAR)) AS INTEGER );
SELECT ('The patch seems to be compatible with schema version '
|| CAST(:expected_version AS VARCHAR)
|| ', applying the patch...') AS compatible_msg;
-- ----------------------------------<actual_patch> -------------------------------------------------
-- part_1: changes to introduce 'SUBMITTED' state of Workers:
ALTER TABLE worker ALTER COLUMN meadow_host DROP NOT NULL;
ALTER TABLE worker ALTER COLUMN meadow_host SET DEFAULT NULL;
ALTER TABLE worker ALTER COLUMN when_born DROP NOT NULL;
ALTER TABLE worker ALTER COLUMN when_born SET DEFAULT NULL;
ALTER TABLE worker ADD COLUMN when_submitted TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
-- part_2: changes to allow extra meadow-specific exit statuses to be representable in the worker table:
ALTER TABLE worker ALTER COLUMN cause_of_death SET DATA TYPE VARCHAR(255);
-- ----------------------------------</actual_patch> -------------------------------------------------
-- increase the schema version by one and register the patch:
UPDATE hive_meta SET meta_value= (CAST(meta_value AS INTEGER) + 1) WHERE meta_key='hive_sql_schema_version';
INSERT INTO hive_meta (meta_key, meta_value) SELECT 'patched_to_' || meta_value, CURRENT_TIMESTAMP FROM hive_meta WHERE meta_key = 'hive_sql_schema_version';
......@@ -493,6 +493,7 @@ CREATE TABLE analysis_data (
@column work_done how many jobs the Worker has completed successfully
@column status current status of the Worker
@column beekeeper_id beekeeper that created this worker
@column when_submitted when the Worker was submitted by a Beekeeper
@column when_born when the Worker process was started
@column when_checked_in when the Worker last checked into the database
@column when_seen when the Worker was last seen by the Meadow
......@@ -505,19 +506,20 @@ CREATE TABLE worker (
worker_id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT,
meadow_type VARCHAR(255) NOT NULL,
meadow_name VARCHAR(255) NOT NULL,
meadow_host VARCHAR(255) NOT NULL,
meadow_host VARCHAR(255) DEFAULT NULL,
meadow_user VARCHAR(255) DEFAULT NULL,
process_id VARCHAR(255) NOT NULL,
resource_class_id INTEGER DEFAULT NULL,
work_done INTEGER NOT NULL DEFAULT 0,
status VARCHAR(255) NOT NULL DEFAULT 'READY', -- expected values: 'SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD'
status VARCHAR(255) NOT NULL DEFAULT 'READY', -- expected values: 'SUBMITTED','SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD'
beekeeper_id INTEGER DEFAULT NULL,
when_born TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_submitted TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_born TIMESTAMP NULL,
when_checked_in TIMESTAMP NULL,
when_seen TIMESTAMP NULL,
when_died TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
cause_of_death ENUM('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN') DEFAULT NULL,
cause_of_death VARCHAR(255) DEFAULT NULL, -- expected values: 'NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN'
log_dir VARCHAR(255) DEFAULT NULL,
KEY meadow_process (meadow_type, meadow_name, process_id)
......
......@@ -506,6 +506,7 @@ CREATE INDEX ON analysis_data (md5sum);
@column work_done how many jobs the Worker has completed successfully
@column status current status of the Worker
@column beekeeper_id beekeeper that created this worker
@column when_submitted when the Worker was submitted by a Beekeeper
@column when_born when the Worker process was started
@column when_checked_in when the Worker last checked into the database
@column when_seen when the Worker was last seen by the Meadow
......@@ -519,18 +520,19 @@ CREATE TABLE worker (
worker_id SERIAL PRIMARY KEY,
meadow_type VARCHAR(255) NOT NULL,
meadow_name VARCHAR(255) NOT NULL,
meadow_host VARCHAR(255) NOT NULL,
meadow_host VARCHAR(255) DEFAULT NULL,
meadow_user VARCHAR(255) DEFAULT NULL,
process_id VARCHAR(255) NOT NULL,
resource_class_id INTEGER DEFAULT NULL,
work_done INTEGER NOT NULL DEFAULT 0,
status VARCHAR(255) NOT NULL DEFAULT 'READY', -- expected values: 'SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD'
status VARCHAR(255) NOT NULL DEFAULT 'READY', -- expected values: 'SUBMITTED','SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD'
beekeeper_id INTEGER DEFAULT NULL,
when_born TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_submitted TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_born TIMESTAMP DEFAULT NULL,
when_checked_in TIMESTAMP DEFAULT NULL,
when_seen TIMESTAMP DEFAULT NULL,
when_died TIMESTAMP DEFAULT NULL,
cause_of_death worker_cod DEFAULT NULL,
cause_of_death VARCHAR(255) DEFAULT NULL, -- expected values: 'NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN'
log_dir VARCHAR(255) DEFAULT NULL
);
CREATE INDEX ON worker (meadow_type, meadow_name, process_id);
......
......@@ -487,6 +487,7 @@ CREATE INDEX analysis_data_idx ON analysis_data (md5sum);
@column work_done how many jobs the Worker has completed successfully
@column status current status of the Worker
@column beekeeper_id beekeeper that created this worker
@column when_submitted when the Worker was submitted by a Beekeeper
@column when_born when the Worker process was started
@column when_checked_in when the Worker last checked into the database
@column when_seen when the Worker was last seen by the Meadow
......@@ -499,18 +500,19 @@ CREATE TABLE worker (
worker_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
meadow_type VARCHAR(255) NOT NULL,
meadow_name VARCHAR(255) NOT NULL,
meadow_host VARCHAR(255) NOT NULL,
meadow_host VARCHAR(255) DEFAULT NULL,
meadow_user VARCHAR(255) DEFAULT NULL,
process_id VARCHAR(255) NOT NULL,
resource_class_id INTEGER DEFAULT NULL,
work_done INTEGER NOT NULL DEFAULT 0,
status VARCHAR(255) NOT NULL DEFAULT 'READY', /* enum('SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD') DEFAULT 'READY' NOT NULL */
status VARCHAR(255) NOT NULL DEFAULT 'READY', /* enum('SUBMITTED','SPECIALIZATION','COMPILATION','READY','JOB_LIFECYCLE','DEAD') DEFAULT 'READY' NOT NULL */
beekeeper_id INTEGER DEFAULT NULL,
when_born TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_submitted TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
when_born TIMESTAMP DEFAULT NULL,
when_checked_in TIMESTAMP DEFAULT NULL,
when_seen TIMESTAMP DEFAULT NULL,
when_died TIMESTAMP DEFAULT NULL,
cause_of_death TEXT DEFAULT NULL, /* enum('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN') DEFAULT NULL */
cause_of_death VARCHAR(255) DEFAULT NULL, /* enum('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN') DEFAULT NULL */
log_dir VARCHAR(255) DEFAULT NULL
);
CREATE INDEX worker_meadow_type_meadow_name_process_id_idx ON worker (meadow_type, meadow_name, process_id);
......
......@@ -144,17 +144,17 @@ my $worker = Bio::EnsEMBL::Hive::Worker->new();
lives_ok( sub {
local $ENV{EHIVE_EXPECTED_BSUB} = '-o /dev/null -e /dev/null -J tracking_homo_sapiens_funcgen_81_38_hive-Hive-/resource_class/-56 /rc_args/ /worker_cmd/';
$lsf_meadow->submit_workers('/worker_cmd/', 1, 56, '/resource_class/', '/rc_args/');
$lsf_meadow->submit_workers_return_meadow_pids('/worker_cmd/', 1, 56, '/resource_class/', '/rc_args/');
}, 'Can submit 1 worker');
lives_ok( sub {
local $ENV{EHIVE_EXPECTED_BSUB} = '-o /dev/null -e /dev/null -J tracking_homo_sapiens_funcgen_81_38_hive-Hive-/resource_class/-56[1-4] /rc_args/ /worker_cmd/';
$lsf_meadow->submit_workers('/worker_cmd/', 4, 56, '/resource_class/', '/rc_args/');
$lsf_meadow->submit_workers_return_meadow_pids('/worker_cmd/', 4, 56, '/resource_class/', '/rc_args/');
}, 'Can submit 4 workers');
lives_ok( sub {
local $ENV{EHIVE_EXPECTED_BSUB} = '-o /submit_log_dir//log_/resource_class/_%J_%I.out -e /submit_log_dir//log_/resource_class/_%J_%I.err -J tracking_homo_sapiens_funcgen_81_38_hive-Hive-/resource_class/-56 /rc_args/ /worker_cmd/';
$lsf_meadow->submit_workers('/worker_cmd/', 1, 56, '/resource_class/', '/rc_args/', '/submit_log_dir/');
$lsf_meadow->submit_workers_return_meadow_pids('/worker_cmd/', 1, 56, '/resource_class/', '/rc_args/', '/submit_log_dir/');
}, 'Can submit 1 worker with a submit_log_dir');
my $expected_bacct = {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment