diff --git a/modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm b/modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm index 0ef80c64393347805d7c36c8d9ff041ec930714b..ac40f97246cc1364a759cde2880b8be0bd652760 100644 --- a/modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm +++ b/modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm @@ -124,14 +124,17 @@ sub find_out_causes { sub submit_workers { - my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_parameters) = @_; + my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name, $rc_parameters, $submit_stdout_file, $submit_stderr_file) = @_; my $job_name = $self->generate_job_name($required_worker_count, $iteration, $rc_name); my $submission_options = $self->config_get('SubmissionOptions'); + $submit_stdout_file ||= '/dev/null'; + $submit_stderr_file ||= '/dev/null'; + $ENV{'LSB_STDOUT_DIRECT'} = 'y'; # unbuffer the output of the bsub command - my $cmd = qq{bsub -o /dev/null -J "${job_name}" $rc_parameters $submission_options $worker_cmd}; + my $cmd = qq{bsub -o $submit_stdout_file -e $submit_stderr_file -J "${job_name}" $rc_parameters $submission_options $worker_cmd}; print "SUBMITTING_CMD:\t\t$cmd\n"; system($cmd) && die "Could not submit job(s): $!, $?"; # let's abort the beekeeper and let the user check the syntax diff --git a/scripts/beekeeper.pl b/scripts/beekeeper.pl index f86ff3a2e2c99c26155c611467b91bfc2823066f..7717566bbc833371d71d6b3a7e47a0019195caf3 100755 --- a/scripts/beekeeper.pl +++ b/scripts/beekeeper.pl @@ -54,7 +54,9 @@ sub main { $self->{'sleep_minutes'} = 1; $self->{'retry_throwing_jobs'} = undef; $self->{'can_respecialize'} = undef; - $self->{'hive_log_dir'} = undef; + $self->{'hive_log_dir'} = undef; + $self->{'submit_stdout_file'} = undef; + $self->{'submit_stderr_file'} = undef; GetOptions( # connection parameters @@ -86,6 +88,8 @@ sub main { 'retry_throwing_jobs=i' => \$self->{'retry_throwing_jobs'}, 'can_respecialize=i' => \$self->{'can_respecialize'}, 'debug=i' => \$self->{'debug'}, + 'submit_stdout_file=s' => \$self->{'submit_stdout_file'}, + 'submit_stderr_file=s' => \$self->{'submit_stderr_file'}, # other commands/options 'h|help' => \$help, @@ -345,7 +349,9 @@ sub run_autonomously { print "Submitting $this_meadow_rc_worker_count workers (rc_name=$rc_name) to ".$this_meadow->signature()."\n"; - $this_meadow->submit_workers($worker_cmd.($special_task ? '' : " -rc_name $rc_name"), $this_meadow_rc_worker_count, $iteration, $rc_name, $rc_name2xparams{ $rc_name } || ''); + $this_meadow->submit_workers($worker_cmd.($special_task ? '' : " -rc_name $rc_name"), $this_meadow_rc_worker_count, $iteration, + $rc_name, $rc_name2xparams{ $rc_name } || '', + $self->{'submit_stdout_file'}, $self->{'submit_stderr_file'}); } } } else { @@ -431,6 +437,8 @@ __DATA__ -total_running_workers_max <num> : max # workers to be running in parallel -submit_workers_max <num> : max # workers to create per loop iteration -submission_options <string> : passes <string> to the Meadow submission command as <options> (formerly lsf_options) + -submit_stdout_file <file> : record submission output stream in a file (to see why workers fail to run after submission). Use with -run instead of -loop. + -submit_stderr_file <file> : record submission error stream in a file (to see why workers fail to run after submission). Use with -run instead of -loop. =head2 Worker control