Skip to content

GitLab

Commit 39726aa3 authored by Leo Gordon's avatar Leo Gordon
Browse files

removed the "compile_module_once" option as the only way to compile modules... 

removed the "compile_module_once" option as the only way to compile modules now is once after specialization
parent c5581fcc
Hide whitespace changes
Inline Side-by-side
Showing with 15 additions and 56 deletions
modules/Bio/EnsEMBL/Hive/Process.pm
View file @ 39726aa3
......@@ -537,28 +537,22 @@ sub debug {
sub worker_temp_directory {
my $self = shift @_;
# TODO: simplify $holding_object=$self when -compile_module_once becomes the only option:
my $holding_object = $self->worker ? $self->worker : $self;
unless(defined($holding_object->{'_tmp_dir'}) and (-e $holding_object->{'_tmp_dir'})) {
unless(defined($self->{'_tmp_dir'}) and (-e $self->{'_tmp_dir'})) {
my $username = $ENV{'USER'};
my $worker_id = $self->worker ? $self->worker->dbID : 'standalone';
$holding_object->{'_tmp_dir'} = "/tmp/worker_${username}.${worker_id}/";
mkdir($holding_object->{'_tmp_dir'}, 0777);
throw("unable to create a writable directory ".$holding_object->{'_tmp_dir'}) unless(-w $holding_object->{'_tmp_dir'});
$self->{'_tmp_dir'} = "/tmp/worker_${username}.${worker_id}/";
mkdir($self->{'_tmp_dir'}, 0777);
throw("unable to create a writable directory ".$self->{'_tmp_dir'}) unless(-w $self->{'_tmp_dir'});
}
return $holding_object->{'_tmp_dir'};
return $self->{'_tmp_dir'};
}
sub cleanup_worker_temp_directory {
my $self = shift @_;
# TODO: simplify $holding_object=$self when -compile_module_once becomes the only option:
my $holding_object = $self->worker ? $self->worker : $self;
if($holding_object->{'_tmp_dir'} and (-e $holding_object->{'_tmp_dir'}) ) {
my $cmd = "rm -r ". $holding_object->{'_tmp_dir'};
if($self->{'_tmp_dir'} and (-e $self->{'_tmp_dir'}) ) {
my $cmd = "rm -r ". $self->{'_tmp_dir'};
system($cmd);
}
}
......
modules/Bio/EnsEMBL/Hive/Queen.pm
View file @ 39726aa3
......@@ -109,10 +109,10 @@ sub create_new_worker {
my ($self, @args) = @_;
my ($meadow_type, $meadow_name, $process_id, $exec_host, $resource_class_id, $resource_class_name,
$no_write, $debug, $worker_log_dir, $hive_log_dir, $job_limit, $life_span, $no_cleanup, $retry_throwing_jobs, $compile_module_once) =
$no_write, $debug, $worker_log_dir, $hive_log_dir, $job_limit, $life_span, $no_cleanup, $retry_throwing_jobs) =
rearrange([qw(meadow_type meadow_name process_id exec_host resource_class_id resource_class_name
no_write debug worker_log_dir hive_log_dir job_limit life_span no_cleanup retry_throwing_jobs compile_module_once) ], @args);
no_write debug worker_log_dir hive_log_dir job_limit life_span no_cleanup retry_throwing_jobs) ], @args);
if( defined($resource_class_name) ) {
my $rc = $self->db->get_ResourceClassAdaptor->fetch_by_name($resource_class_name)
......@@ -162,8 +162,6 @@ sub create_new_worker {
$worker->retry_throwing_jobs($retry_throwing_jobs) if(defined $retry_throwing_jobs);
$worker->compile_module_once($compile_module_once) if(defined $compile_module_once);
return $worker;
}
......
modules/Bio/EnsEMBL/Hive/Worker.pm
View file @ 39726aa3
......@@ -413,13 +413,6 @@ sub retry_throwing_jobs {
return defined($self->{'_retry_throwing_jobs'}) ? $self->{'_retry_throwing_jobs'} : 1;
}
sub compile_module_once {
my $self = shift @_;
$self->{'_compile_module_once'} = shift @_ if(@_);
return defined($self->{'_compile_module_once'}) ? $self->{'_compile_module_once'} : 1;
}
sub get_stdout_redirector {
my $self = shift;
......@@ -469,14 +462,6 @@ sub toString {
}
sub cleanup_worker_process_temp_directory {
my $self = shift;
if($self->{'_tmp_dir'} and (-e $self->{'_tmp_dir'}) ) {
my $cmd = "rm -r ". $self->{'_tmp_dir'};
system($cmd);
}
}
###############################
#
# WORK section
......@@ -530,7 +515,7 @@ sub run {
$self->cause_of_death('SEE_MSG') unless($self->cause_of_death()); # some specific causes could have been set prior to die "...";
};
if(!$self->cause_of_death() and $self->compile_module_once() ) {
if( !$self->cause_of_death() ) {
eval {
$self->enter_status('COMPILATION');
my $runnable_object = $self->analysis->process or die "Unknown compilation error";
......@@ -616,10 +601,8 @@ sub run {
# have runnable clean up any global/process files/data it may have created
if($self->perform_cleanup) {
if(my $runnable_object = $self->runnable_object()) { # if -compile_module_once is 1, keep _tmp_dir in the Process object:
if(my $runnable_object = $self->runnable_object()) { # the temp_directory is actually kept in the Process object:
$runnable_object->cleanup_worker_temp_directory();
} else { # otherwise keep _tmp_dir in the Worker object, so it needs its own cleanup method:
$self->cleanup_worker_process_temp_directory(); # TODO: remove this method when -compile_module_once becomes the only option
}
}
......@@ -636,6 +619,7 @@ sub run {
}
}
sub run_one_batch {
my ($self, $jobs) = @_;
......@@ -665,18 +649,7 @@ sub run_one_batch {
eval { # capture any throw/die
$job->incomplete(1);
my $runnable_object;
if( $self->compile_module_once() ) {
$runnable_object = $self->runnable_object();
} else {
$self->enter_status('COMPILATION', $job);
$runnable_object = $self->analysis->process or die "Unknown compilation error";
$runnable_object->db( $self->adaptor->db );
$runnable_object->worker( $self );
$runnable_object->debug( $self->debug );
$runnable_object->execute_writes( $self->execute_writes );
}
my $runnable_object = $self->runnable_object();
$self->adaptor->db->dbc->query_count(0);
$job_stopwatch->restart();
......
scripts/beekeeper.pl
View file @ 39726aa3
......@@ -53,7 +53,6 @@ sub main {
$self->{'sleep_minutes'} = 1;
$self->{'retry_throwing_jobs'} = undef;
$self->{'compile_module_once'} = undef;
$self->{'hive_log_dir'} = undef;
GetOptions(
......@@ -84,7 +83,6 @@ sub main {
'logic_name=s' => \$self->{'logic_name'},
'hive_log_dir|hive_output_dir=s' => \$self->{'hive_log_dir'},
'retry_throwing_jobs=i' => \$self->{'retry_throwing_jobs'},
'compile_module_once=i' => \$self->{'compile_module_once'},
'debug=i' => \$self->{'debug'},
# other commands/options
......@@ -279,7 +277,7 @@ sub generate_worker_cmd {
$worker_cmd .= " -url '". $self->{'safe_url'} ."'";
}
foreach my $worker_option ('job_limit', 'life_span', 'retry_throwing_jobs', 'compile_module_once', 'hive_log_dir', 'debug') {
foreach my $worker_option ('job_limit', 'life_span', 'retry_throwing_jobs', 'hive_log_dir', 'debug') {
if(defined(my $value = $self->{$worker_option})) {
$worker_cmd .= " -${worker_option} $value";
}
......@@ -438,7 +436,6 @@ __DATA__
-life_span <num> : life_span limit for each worker
-logic_name <string> : restrict the pipeline stat/runs to this analysis logic_name
-retry_throwing_jobs 0|1 : if a job dies *knowingly*, should we retry it by default?
-compile_module_once 0|1 : should we compile the module only once (desired future behaviour), or pretend to do it before every job (current behaviour)?
-hive_log_dir <path> : directory where stdout/stderr of the hive is redirected
-debug <debug_level> : set debug level of the workers
......
scripts/runWorker.pl
View file @ 39726aa3
......@@ -13,7 +13,7 @@ Bio::EnsEMBL::Registry->no_version_check(1);
my ($reg_conf, $reg_alias, $url); # Connection parameters
my ($resource_class_id, $resource_class_name, $analysis_id, $logic_name, $job_id); # Task specification parameters
my ($job_limit, $life_span, $no_cleanup, $no_write, $hive_log_dir, $worker_log_dir, $retry_throwing_jobs, $compile_module_once, $force); # Worker control parameters
my ($job_limit, $life_span, $no_cleanup, $no_write, $hive_log_dir, $worker_log_dir, $retry_throwing_jobs, $force); # Worker control parameters
my ($help, $debug);
GetOptions(
......@@ -38,7 +38,6 @@ GetOptions(
'hive_log_dir|hive_output_dir=s' => \$hive_log_dir, # keep compatibility with the old name
'worker_log_dir|worker_output_dir=s' => \$worker_log_dir, # will take precedence over hive_log_dir if set
'retry_throwing_jobs=i' => \$retry_throwing_jobs,
'compile_module_once=i' => \$compile_module_once,
'force=i' => \$force,
# Other commands
......@@ -101,7 +100,6 @@ eval {
-worker_log_dir => $worker_log_dir,
-hive_log_dir => $hive_log_dir,
-retry_throwing_jobs => $retry_throwing_jobs,
-compile_module_once => $compile_module_once,
# Other parameters:
-debug => $debug,
......@@ -185,7 +183,6 @@ __DATA__
-hive_log_dir <path> : directory where stdout/stderr of the whole hive of workers is redirected
-worker_log_dir <path> : directory where stdout/stderr of this particular worker is redirected
-retry_throwing_jobs <0|1> : if a job dies *knowingly*, should we retry it by default?
-compile_module_once 0|1 : should we compile the module only once (desired future behaviour), or pretend to do it before every job (current behaviour)?
-force 0|1 : set to 1 if you want to force running a Worker over a BLOCKED analysis or to run a specific DONE/SEMAPHORED job_id
=head2 Other options:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment