Skip to content
Snippets Groups Projects
Commit 1ca856a8 authored by Leo Gordon's avatar Leo Gordon
Browse files

remove the per-worker batch_size flag from scripts

parent 76b1900d
No related branches found
No related tags found
No related merge requests found
......@@ -94,13 +94,14 @@ sub main {
# worker control
'job_limit|jlimit=i' => \$self->{'job_limit'},
'batch_size=i' => \$self->{'batch_size'},
'life_span|lifespan=i' => \$self->{'life_span'},
'logic_name=s' => \$self->{'logic_name'},
'hive_output_dir=s' => \$self->{'hive_output_dir'},
'maximise_concurrency=i' => \$self->{'maximise_concurrency'},
'retry_throwing_jobs=i' => \$self->{'retry_throwing_jobs'},
'batch_size=i' => \$self->{'batch_size'}, # OBSOLETE!
# other commands/options
'h|help' => \$help,
'sync' => \$sync,
......@@ -123,6 +124,11 @@ sub main {
if ($help) { script_usage(0); }
if( $self->{'batch_size'} ) {
print "\nERROR : -batch_size flag is obsolete, please modify batch_size of the analysis instead\n";
script_usage(1);
}
parse_conf($self, $conf_file);
if($run or $self->{'run_job_id'}) {
......@@ -333,7 +339,7 @@ sub generate_worker_cmd {
if ($self->{'run_job_id'}) {
$worker_cmd .= " -job_id ".$self->{'run_job_id'};
} else {
foreach my $worker_option ('batch_size', 'job_limit', 'life_span', 'logic_name', 'maximize_concurrency', 'retry_throwing_jobs', 'hive_output_dir') {
foreach my $worker_option ('job_limit', 'life_span', 'logic_name', 'maximize_concurrency', 'retry_throwing_jobs', 'hive_output_dir') {
if(defined(my $value = $self->{$worker_option})) {
$worker_cmd .= " -${worker_option} $value";
}
......@@ -510,13 +516,14 @@ __DATA__
=head2 Worker control
-job_limit <num> : #jobs to run before worker can die naturally
-batch_size <num> : #jobs a worker can claim at once
-life_span <num> : life_span limit for each worker
-logic_name <string> : restrict the pipeline stat/runs to this analysis logic_name
-maximise_concurrency 1 : try to run more different analyses at the same time
-retry_throwing_jobs 0|1 : if a job dies *knowingly*, should we retry it by default?
-hive_output_dir <path> : directory where stdout/stderr of the hive is redirected
-batch_size <num> : [OBSOLETE!] Please modify batch_size of the analysis instead
=head2 Other commands/options
-help : print this help
......
......@@ -45,7 +45,6 @@ GetOptions(
'job_id=i' => \$job_id,
# Worker control parameters:
'batch_size=i' => \$batch_size,
'job_limit|limit=i' => \$job_limit,
'life_span|lifespan=i' => \$life_span,
'no_cleanup' => \$no_cleanup,
......@@ -54,6 +53,8 @@ GetOptions(
'worker_output_dir=s' => \$worker_output_dir, # will take precedence over hive_output_dir if set
'retry_throwing_jobs=i' => \$retry_throwing_jobs,
'batch_size=i' => \$batch_size, # OBSOLETE!
# Other commands
'h|help' => \$help,
'debug=i' => \$debug,
......@@ -66,6 +67,12 @@ GetOptions(
if ($help) { script_usage(0); }
if( $batch_size ) {
print "\nERROR : -batch_size flag is obsolete, please modify batch_size of the analysis instead\n";
script_usage(1);
}
parse_conf($conf_file);
if($reg_conf) { # if reg_conf is defined, we load it regardless of whether it is used to connect to the Hive database or not:
......@@ -113,7 +120,6 @@ eval {
-job_id => $job_id,
# Worker control parameters:
-batch_size => $batch_size,
-job_limit => $job_limit,
-life_span => $life_span,
-no_cleanup => $no_cleanup,
......@@ -122,6 +128,9 @@ eval {
-hive_output_dir => $hive_output_dir,
-retry_throwing_jobs => $retry_throwing_jobs,
## Obsolete:
# -batch_size => $batch_size,
# Other parameters:
-debug => $debug,
);
......@@ -229,7 +238,6 @@ __DATA__
=head2 Worker control parameters:
-batch_size <num> : #jobs to claim at a time
-job_limit <num> : #jobs to run before worker can die naturally
-life_span <num> : number of minutes this worker is allowed to run
-no_cleanup : don't perform temp directory cleanup when worker exits
......@@ -238,6 +246,8 @@ __DATA__
-worker_output_dir <path> : directory where stdout/stderr of this particular worker is redirected
-retry_throwing_jobs <0|1> : if a job dies *knowingly*, should we retry it by default?
-batch_size <num> : [OBSOLETE!] Please modify batch_size of the analysis instead
=head2 Other options:
-help : print this help
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment