beekeeper.pl 20.6 KB
Newer Older
1
#!/usr/bin/env perl
2 3

use strict;
Leo Gordon's avatar
Leo Gordon committed
4
use warnings;
5
use Data::Dumper;
6
use Getopt::Long;
Leo Gordon's avatar
Leo Gordon committed
7

8
use Bio::EnsEMBL::Hive::Utils ('script_usage', 'destringify');
9 10
use Bio::EnsEMBL::Hive::Utils::Config;
use Bio::EnsEMBL::Hive::URLFactory;
11 12
use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
use Bio::EnsEMBL::Hive::Queen;
13
use Bio::EnsEMBL::Hive::Valley;
14

Leo Gordon's avatar
Leo Gordon committed
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
main();

sub main {
    $| = 1;
    Bio::EnsEMBL::Registry->no_version_check(1);

        # ok this is a hack, but I'm going to pretend I've got an object here
        # by creating a hash ref and passing it around like an object
        # this is to avoid using global variables in functions, and to consolidate
        # the globals into a nice '$self' package
    my $self = {};

    $self->{'db_conf'} = {
        -host   => '',
        -port   => 3306,
        -user   => 'ensro',
        -pass   => '',
        -dbname => '',
    };

35
    my $help;
Leo Gordon's avatar
Leo Gordon committed
36 37 38 39
    my $loopit                      = 0;
    my $sync                        = 0;
    my $local                       = 0;
    my $show_failed_jobs            = 0;
40
    my $meadow_type                 = undef;
41 42 43 44
    my $pending_adjust              = undef;
    my $submit_workers_max          = undef;
    my $total_running_workers_max   = undef;
    my $submission_options          = undef;
Leo Gordon's avatar
Leo Gordon committed
45
    my $run                         = 0;
46
    my $max_loops                   = 0; # not running by default
47
    my $run_job_id                  = undef;
48
    my $keep_alive                  = 0; # ==1 means run even when there is nothing to do
Leo Gordon's avatar
Leo Gordon committed
49 50 51 52 53 54 55
    my $check_for_dead              = 0;
    my $all_dead                    = 0;
    my $job_id_for_output           = 0;
    my $show_worker_stats           = 0;
    my $kill_worker_id              = 0;
    my $reset_job_id                = 0;
    my $reset_all_jobs_for_analysis = 0;
56
    my $reset_failed_jobs_for_analysis = 0;
Leo Gordon's avatar
Leo Gordon committed
57

58 59 60
    $self->{'reg_conf'}             = undef;
    $self->{'reg_alias'}            = undef;

Leo Gordon's avatar
Leo Gordon committed
61
    $self->{'sleep_minutes'}        = 1;
Leo Gordon's avatar
Leo Gordon committed
62
    $self->{'verbose_stats'}        = 1;
63
    $self->{'retry_throwing_jobs'}  = undef;
64
    $self->{'compile_module_once'}  = undef;
65
    $self->{'hive_output_dir'} = undef;
Leo Gordon's avatar
Leo Gordon committed
66

Leo Gordon's avatar
Leo Gordon committed
67
    GetOptions(
Leo Gordon's avatar
Leo Gordon committed
68
                    # connection parameters
69 70 71 72 73 74 75 76
               'reg_conf|regfile=s' => \$self->{'reg_conf'},
               'reg_alias|regname=s'=> \$self->{'reg_alias'},
               'url=s'              => \$self->{'url'},
               'host|dbhost=s'      => \$self->{'db_conf'}->{'-host'},
               'port|dbport=i'      => \$self->{'db_conf'}->{'-port'},
               'user|dbuser=s'      => \$self->{'db_conf'}->{'-user'},
               'password|dbpass=s'  => \$self->{'db_conf'}->{'-pass'},
               'database|dbname=s'  => \$self->{'db_conf'}->{'-dbname'},
Leo Gordon's avatar
Leo Gordon committed
77 78

                    # loop control
79 80 81 82
               'run'                => \$run,
               'loop'               => \$loopit,
               'max_loops=i'        => \$max_loops,
               'keep_alive'         => \$keep_alive,
83
               'job_id|run_job_id=i'=> \$run_job_id,
84
               'sleep=f'            => \$self->{'sleep_minutes'},
Leo Gordon's avatar
Leo Gordon committed
85 86

                    # meadow control
87 88 89 90 91 92
               'local!'                         => \$local,
               'meadow_type=s'                  => \$meadow_type,
               'total_running_workers_max=i'    => \$total_running_workers_max,
               'submit_workers_max=i'           => \$submit_workers_max,
               'pending_adjust=i'               => \$pending_adjust,
               'submission_options=s'           => \$submission_options,
Leo Gordon's avatar
Leo Gordon committed
93 94

                    # worker control
95 96
               'job_limit|jlimit=i'     => \$self->{'job_limit'},
               'life_span|lifespan=i'   => \$self->{'life_span'},
97 98
               'logic_name=s'           => \$self->{'logic_name'},
               'hive_output_dir=s'      => \$self->{'hive_output_dir'},
99
               'retry_throwing_jobs=i'  => \$self->{'retry_throwing_jobs'},
100
               'compile_module_once=i'  => \$self->{'compile_module_once'},
101
               'debug=i'                => \$self->{'debug'},
Leo Gordon's avatar
Leo Gordon committed
102 103

                    # other commands/options
Leo Gordon's avatar
Leo Gordon committed
104
               'h|help'            => \$help,
Leo Gordon's avatar
Leo Gordon committed
105 106 107 108 109 110 111 112 113
               'sync'              => \$sync,
               'dead'              => \$check_for_dead,
               'killworker=i'      => \$kill_worker_id,
               'alldead'           => \$all_dead,
               'no_analysis_stats' => \$self->{'no_analysis_stats'},
               'verbose_stats=i'   => \$self->{'verbose_stats'},
               'worker_stats'      => \$show_worker_stats,
               'failed_jobs'       => \$show_failed_jobs,
               'reset_job_id=i'    => \$reset_job_id,
114
               'reset_failed|reset_failed_jobs_for_analysis=s' => \$reset_failed_jobs_for_analysis,
Leo Gordon's avatar
Leo Gordon committed
115 116 117
               'reset_all|reset_all_jobs_for_analysis=s' => \$reset_all_jobs_for_analysis,
               'job_output=i'      => \$job_id_for_output,
               'monitor!'          => \$self->{'monitor'},
118 119 120

                    # loose arguments interpreted as database name (for compatibility with mysql[dump])
               '<>', sub { $self->{'db_conf'}->{'-dbname'} = shift @_; },
Leo Gordon's avatar
Leo Gordon committed
121 122
    );

123
    if ($help) { script_usage(0); }
Leo Gordon's avatar
Leo Gordon committed
124

125 126
    my $config = Bio::EnsEMBL::Hive::Utils::Config->new();      # will probably add a config_file option later

127
    if($run or $run_job_id) {
Leo Gordon's avatar
Leo Gordon committed
128
        $max_loops = 1;
129
    } elsif ($loopit or $keep_alive) {
Leo Gordon's avatar
Leo Gordon committed
130 131 132 133 134 135 136
        unless($max_loops) {
            $max_loops = -1; # unlimited
        }
        unless(defined($self->{'monitor'})) {
            $self->{'monitor'} = 1;
        }
    }
137

138 139 140
    if($self->{'reg_conf'} and $self->{'reg_alias'}) {
        Bio::EnsEMBL::Registry->load_all($self->{'reg_conf'});
        $self->{'dba'} = Bio::EnsEMBL::Registry->get_DBAdaptor($self->{'reg_alias'}, 'hive');
Leo Gordon's avatar
Leo Gordon committed
141 142 143 144 145 146 147 148
    } elsif($self->{'url'}) {
        $self->{'dba'} = Bio::EnsEMBL::Hive::URLFactory->fetch($self->{'url'}) || die("Unable to connect to $self->{'url'}\n");
    } elsif (    $self->{'db_conf'}->{'-host'}
             and $self->{'db_conf'}->{'-user'}
             and $self->{'db_conf'}->{'-dbname'}) { # connect to database specified
                    $self->{'dba'} = new Bio::EnsEMBL::Hive::DBSQL::DBAdaptor(%{$self->{'db_conf'}});
                    $self->{'url'} = $self->{'dba'}->dbc->url;
    } else {
149
        print "\nERROR : Connection parameters (reg_conf+reg_alias, url or dbhost+dbuser+dbname) need to be specified\n\n";
150
        script_usage(1);
Leo Gordon's avatar
Leo Gordon committed
151
    }
152

Leo Gordon's avatar
Leo Gordon committed
153 154
    my $queen = $self->{'dba'}->get_Queen;
    $queen->{'verbose_stats'} = $self->{'verbose_stats'};
155

156 157 158 159
    my $pipeline_name = destringify(
            $self->{'dba'}->get_MetaContainer->list_value_by_key("pipeline_name")->[0]
         || $self->{'dba'}->get_MetaContainer->list_value_by_key("name")->[0]
    );
160

161 162 163 164 165 166 167 168 169 170 171 172 173 174
    unless($pipeline_name) {
        print STDERR "+---------------------------------------------------------------------+\n";
        print STDERR "!                                                                     !\n";
        print STDERR "!                  WARNING:                                           !\n";
        print STDERR "!                                                                     !\n";
        print STDERR "! At the moment your pipeline doesn't have 'pipeline_name' defined.   !\n";
        print STDERR "! This may seriously impair your beekeeping experience unless you are !\n";
        print STDERR "! the only farm user. The name should be set in your PipeConfig file, !\n";
        print STDERR "! or if you are running an old pipeline you can just set it by hand   !\n";
        print STDERR "! in the 'meta' table.                                                !\n";
        print STDERR "!                                                                     !\n";
        print STDERR "+---------------------------------------------------------------------+\n";
    }

175 176 177 178
    if($run_job_id) {
        $submit_workers_max = 1;
    }

179
    $meadow_type = 'LOCAL' if($local);
180
    my $valley = Bio::EnsEMBL::Hive::Valley->new( $config, $meadow_type, $pipeline_name );
181

182
    my $current_meadow = $valley->get_current_meadow();
Leo Gordon's avatar
Leo Gordon committed
183
    warn "Current ".$current_meadow->toString."\n\n";
184

185 186 187 188
    $current_meadow->config_set('TotalRunningWorkersMax', $total_running_workers_max) if(defined $total_running_workers_max);
    $current_meadow->config_set('PendingAdjust', $pending_adjust) if(defined $pending_adjust);
    $current_meadow->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
    $current_meadow->config_set('SubmissionOptions', $submission_options) if(defined $submission_options);
189

190
    if($reset_job_id) { $queen->reset_job_by_dbID_and_sync($reset_job_id); }
191

Leo Gordon's avatar
Leo Gordon committed
192 193 194 195 196
    if($job_id_for_output) {
        printf("===== job output\n");
        my $job = $self->{'dba'}->get_AnalysisJobAdaptor->fetch_by_dbID($job_id_for_output);
        $job->print_job();
    }
197

198 199 200 201 202 203 204
    if(my $logic_name_to_reset = $reset_all_jobs_for_analysis || $reset_failed_jobs_for_analysis) {

        my $analysis = $self->{'dba'}->get_AnalysisAdaptor->fetch_by_logic_name($logic_name_to_reset)
              || die( "Cannot AnalysisAdaptor->fetch_by_logic_name($logic_name_to_reset)"); 

        $self->{'dba'}->get_AnalysisJobAdaptor->reset_jobs_for_analysis_id($analysis->dbID, $reset_all_jobs_for_analysis); 
        $self->{'dba'}->get_Queen->synchronize_AnalysisStats($analysis->stats);
Leo Gordon's avatar
Leo Gordon committed
205
    }
206

Leo Gordon's avatar
Leo Gordon committed
207
    if($all_dead)           { $queen->register_all_workers_dead(); }
208
    if($check_for_dead)     { $queen->check_for_dead_workers($valley, 1); }
Javier Herrero's avatar
Javier Herrero committed
209

Leo Gordon's avatar
Leo Gordon committed
210
    if ($kill_worker_id) {
211
        my $worker = $queen->fetch_by_dbID($kill_worker_id);
Javier Herrero's avatar
Javier Herrero committed
212

213 214 215
        unless( $worker->cause_of_death() ) {
            if( my $meadow = $valley->find_available_meadow_responsible_for_worker( $worker ) ) {

216 217 218 219 220 221 222 223 224 225 226 227
                if( $meadow->check_worker_is_alive_and_mine ) {
                    printf("Killing worker: %10d %35s %15s  %20s(%d) : ", 
                            $worker->dbID, $worker->host, $worker->process_id, 
                            $worker->analysis->logic_name, $worker->analysis->dbID);

                    $meadow->kill_worker($worker);
                    $worker->cause_of_death('KILLED_BY_USER');
                    $queen->register_worker_death($worker);
                         # what about clean-up? Should we do it here or not?
                } else {
                    die "According to the Meadow, the Worker (dbID=$kill_worker_id) is not running, so cannot kill";
                }
228
            } else {
229
                die "Cannot access the Meadow responsible for the Worker (dbID=$kill_worker_id), so cannot kill";
230 231
            }
        } else {
232
            die "According to the Queen, the Worker (dbID=$kill_worker_id) is not running, so cannot kill";
Leo Gordon's avatar
Leo Gordon committed
233 234
        }
    }
Javier Herrero's avatar
Javier Herrero committed
235

Leo Gordon's avatar
Leo Gordon committed
236
    my $analysis = $self->{'dba'}->get_AnalysisAdaptor->fetch_by_logic_name($self->{'logic_name'});
237

Leo Gordon's avatar
Leo Gordon committed
238
    if ($max_loops) { # positive $max_loop means limited, negative means unlimited
239

240
        run_autonomously($self, $max_loops, $keep_alive, $queen, $valley, $analysis, $run_job_id);
Javier Herrero's avatar
Javier Herrero committed
241

Leo Gordon's avatar
Leo Gordon committed
242 243
    } else {
            # the output of several methods will look differently depending on $analysis being [un]defined
244

Leo Gordon's avatar
Leo Gordon committed
245 246 247 248
        if($sync) {
            $queen->synchronize_hive($analysis);
        }
        $queen->print_analysis_status($analysis) unless($self->{'no_analysis_stats'});
249

250
        if($show_worker_stats) {
Leo Gordon's avatar
Leo Gordon committed
251
            print "\n===== List of live Workers according to the Queen: ======\n";
252 253 254 255
            foreach my $worker (@{ $queen->fetch_overdue_workers(0) }) {
                print $worker->toString()."\n";
            }
        }
Leo Gordon's avatar
Leo Gordon committed
256
        $queen->print_running_worker_counts;
257

258
        $queen->schedule_workers($analysis);    # show what would be submitted, but do not actually submit
Leo Gordon's avatar
Leo Gordon committed
259
        $queen->get_remaining_jobs_show_hive_progress();
260

Leo Gordon's avatar
Leo Gordon committed
261 262 263
        if($show_failed_jobs) {
            print("===== failed jobs\n");
            my $failed_job_list = $self->{'dba'}->get_AnalysisJobAdaptor->fetch_all_failed_jobs();
264

Leo Gordon's avatar
Leo Gordon committed
265 266 267 268 269
            foreach my $job (@{$failed_job_list}) {
                $job->print_job();
            }
        }
    }
Javier Herrero's avatar
Javier Herrero committed
270

Leo Gordon's avatar
Leo Gordon committed
271 272 273
    if ($self->{'monitor'}) {
        $queen->monitor();
    }
274

Leo Gordon's avatar
Leo Gordon committed
275
    exit(0);
276 277
}

278 279 280 281 282 283 284
#######################
#
# subroutines
#
#######################


Leo Gordon's avatar
Leo Gordon committed
285
sub generate_worker_cmd {
286
    my ($self, $run_job_id) = @_;
287

288 289 290 291 292 293 294 295 296 297 298 299
    my $worker_cmd = 'runWorker.pl';

    if ($self->{'reg_conf'}) {      # if reg_conf is defined, we have to pass it anyway, regardless of whether it is used to connect to the Hive database or not:
        $worker_cmd .= ' -reg_conf '. $self->{'reg_conf'};
    }

    if ($self->{'reg_alias'}) {     # then we pass the connection parameters:
        $worker_cmd .= ' -reg_alias '. $self->{'reg_alias'};
    } else {
        $worker_cmd .= ' -url '. $self->{'url'};
    }

300 301
    if ($run_job_id) {
        $worker_cmd .= " -job_id $run_job_id";
Leo Gordon's avatar
Leo Gordon committed
302
    } else {
303
        foreach my $worker_option ('job_limit', 'life_span', 'logic_name', 'retry_throwing_jobs', 'compile_module_once', 'hive_output_dir', 'debug') {
304 305 306 307
            if(defined(my $value = $self->{$worker_option})) {
                $worker_cmd .= " -${worker_option} $value";
            }
        }
Leo Gordon's avatar
Leo Gordon committed
308
    }
309

Leo Gordon's avatar
Leo Gordon committed
310
    return $worker_cmd;
311 312 313
}

sub run_autonomously {
314
    my ($self, $max_loops, $keep_alive, $queen, $valley, $this_analysis, $run_job_id) = @_;
315

Leo Gordon's avatar
Leo Gordon committed
316 317 318
    unless(`runWorker.pl`) {
        print("can't find runWorker.pl script.  Please make sure it's in your path\n");
        exit(1);
319 320
    }

321
    my $current_meadow = $valley->get_current_meadow();
322
    my $worker_cmd = generate_worker_cmd($self, $run_job_id);
Leo Gordon's avatar
Leo Gordon committed
323

Leo Gordon's avatar
Leo Gordon committed
324
        # pre-hash the resource_class xparams for future use:
325
    my $rc_xparams = $self->{'dba'}->get_ResourceDescriptionAdaptor->fetch_by_meadow_type_HASHED_FROM_resource_class_id_TO_parameters($current_meadow->type());
Leo Gordon's avatar
Leo Gordon committed
326

Leo Gordon's avatar
Leo Gordon committed
327 328 329 330 331 332 333 334 335
    my $iteration=0;
    my $num_of_remaining_jobs=0;
    my $failed_analyses=0;
    do {
        if($iteration++) {
            $queen->monitor();
            $self->{'dba'}->dbc->disconnect_if_idle;
            printf("sleep %.2f minutes. Next loop at %s\n", $self->{'sleep_minutes'}, scalar localtime(time+$self->{'sleep_minutes'}*60));
            sleep($self->{'sleep_minutes'}*60);  
336
        }
337

Leo Gordon's avatar
Leo Gordon committed
338
        print("\n======= beekeeper loop ** $iteration **==========\n");
Javier Herrero's avatar
Javier Herrero committed
339

340
        $queen->check_for_dead_workers($valley, 0);
341

Leo Gordon's avatar
Leo Gordon committed
342
        $queen->print_analysis_status unless($self->{'no_analysis_stats'});
Leo Gordon's avatar
Leo Gordon committed
343
        $queen->print_running_worker_counts;
Leo Gordon's avatar
Leo Gordon committed
344

345
        my $workers_to_run_by_rc_id = $queen->schedule_workers_resync_if_necessary($valley, $this_analysis);
346

347
        if(keys %$workers_to_run_by_rc_id) {
348 349
            foreach my $rc_id ( sort { $workers_to_run_by_rc_id->{$a}<=>$workers_to_run_by_rc_id->{$b} } keys %$workers_to_run_by_rc_id) {
                my $this_rc_worker_count = $workers_to_run_by_rc_id->{$rc_id};
350

351
                print "Submitting $this_rc_worker_count workers (rc_id=$rc_id) to ".$current_meadow->toString()."\n";
Leo Gordon's avatar
Leo Gordon committed
352

353
                $current_meadow->submit_workers($iteration, $worker_cmd, $this_rc_worker_count, $rc_id, $rc_xparams->{$rc_id} || '');
Leo Gordon's avatar
Leo Gordon committed
354
            }
Leo Gordon's avatar
Leo Gordon committed
355 356 357
        } else {
            print "Not submitting any workers this iteration\n";
        }
358

Leo Gordon's avatar
Leo Gordon committed
359
        $failed_analyses       = $queen->get_num_failed_analyses($this_analysis);
Leo Gordon's avatar
Leo Gordon committed
360
        $num_of_remaining_jobs = $queen->get_remaining_jobs_show_hive_progress();
361

362 363
    } while( $keep_alive
            or (!$failed_analyses and $num_of_remaining_jobs and $iteration!=$max_loops) );
364

Leo Gordon's avatar
Leo Gordon committed
365 366 367 368 369 370
    print "The Beekeeper has stopped because ".(
          $failed_analyses ? "there were $failed_analyses failed analyses"
        : !$num_of_remaining_jobs ? "there is nothing left to do"
        : "the number of loops was limited by $max_loops and this limit expired"
    )."\n";

Leo Gordon's avatar
Leo Gordon committed
371
    printf("dbc %d disconnect cycles\n", $self->{'dba'}->dbc->disconnect_count);
372 373
}

Leo Gordon's avatar
Leo Gordon committed
374

375 376 377 378 379 380 381 382 383 384
__DATA__

=pod

=head1 NAME

    beekeeper.pl

=head1 DESCRIPTION

Leo Gordon's avatar
Leo Gordon committed
385 386 387
    The Beekeeper is in charge of interfacing between the Queen and a compute resource or 'compute farm'.
    Its job is to initialize/sync the eHive database (via the Queen), query the Queen if it needs any workers
    and to send the requested number of workers to open machines via the runWorker.pl script.
Leo Gordon's avatar
Leo Gordon committed
388

Leo Gordon's avatar
Leo Gordon committed
389
    It is also responsible for interfacing with the Queen to identify workers which died
390
    unexpectedly so that she can free the dead workers and reclaim unfinished jobs.
391 392 393

=head1 USAGE EXAMPLES

Leo Gordon's avatar
Leo Gordon committed
394
        # Usually run after the pipeline has been created to calculate the internal statistics necessary for eHive functioning
395
    beekeeper.pl --host=hostname --port=3306 --user=username --password=secret ehive_dbname -sync
396

Leo Gordon's avatar
Leo Gordon committed
397 398
        # An alternative way of doing the same thing
    beekeeper.pl -url mysql://username:secret@hostname:port/ehive_dbname -sync
399

Leo Gordon's avatar
Leo Gordon committed
400 401
        # Do not run any additional Workers, just check for the current status of the pipeline:
    beekeeper.pl -url mysql://username:secret@hostname:port/ehive_dbname
Leo Gordon's avatar
Leo Gordon committed
402

403 404
        # Run the pipeline in automatic mode (-loop), run all the workers locally (-meadow_type LOCAL) and allow for 3 parallel workers (-total_running_workers_max 3)
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -meadow_type LOCAL -total_running_workers_max 3 -loop
405

Leo Gordon's avatar
Leo Gordon committed
406 407
        # Run in automatic mode, but only restrict to running the 'fast_blast' analysis
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -logic_name fast_blast -loop
408

Leo Gordon's avatar
Leo Gordon committed
409 410
        # Restrict the normal execution to one iteration only - can be used for testing a newly set up pipeline
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -run
411

412 413
        # Reset failed 'buggy_analysis' jobs to 'READY' state, so that they can be run again
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -reset_failed_jobs_for_analysis buggy_analysis
414

Leo Gordon's avatar
Leo Gordon committed
415 416
        # Do a cleanup: find and bury dead workers, reclaim their jobs
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -dead
417 418 419 420 421

=head1 OPTIONS

=head2 Connection parameters

422 423
    -reg_conf <path>       : path to a Registry configuration file
    -reg_alias <string>    : species/alias name for the Hive DBAdaptor
Leo Gordon's avatar
Leo Gordon committed
424 425 426 427 428
    -url <url string>      : url defining where hive database is located
    -host <machine>        : mysql database host <machine>
    -port <port#>          : mysql port number
    -user <name>           : mysql connection user <name>
    -password <pass>       : mysql connection password <pass>
429
    [-database] <name>     : mysql database <name>
430 431 432

=head2 Looping control

Leo Gordon's avatar
Leo Gordon committed
433 434
    -loop                  : run autonomously, loops and sleeps
    -max_loops <num>       : perform max this # of loops in autonomous mode
435
    -keep_alive            : do not stop when there are no more jobs to do - carry on looping
Leo Gordon's avatar
Leo Gordon committed
436
    -job_id <job_id>       : run 1 iteration for this job_id
437
    -run                   : run 1 iteration of automation loop
Leo Gordon's avatar
Leo Gordon committed
438
    -sleep <num>           : when looping, sleep <num> minutes (default 2min)
439

440
=head2 Current Meadow control
441

442 443 444 445 446
    -meadow_type <string>               : the desired Meadow class name, such as 'LSF' or 'LOCAL'
    -total_running_workers_max <num>    : max # workers to be running in parallel
    -submit_workers_max <num>           : max # workers to create per loop iteration
    -pending_adjust <0|1>               : [do not] adjust needed workers by pending workers
    -submission_options <string>        : passes <string> to the Meadow submission command as <options> (formerly lsf_options)
447 448 449

=head2 Worker control

450
    -job_limit <num>            : #jobs to run before worker can die naturally
451
    -life_span <num>            : life_span limit for each worker
452 453
    -logic_name <string>        : restrict the pipeline stat/runs to this analysis logic_name
    -retry_throwing_jobs 0|1    : if a job dies *knowingly*, should we retry it by default?
454
    -compile_module_once 0|1    : should we compile the module only once (desired future behaviour), or pretend to do it before every job (current behaviour)?
455
    -hive_output_dir <path>     : directory where stdout/stderr of the hive is redirected
456
    -debug <debug_level>        : set debug level of the workers
457 458 459

=head2 Other commands/options

Leo Gordon's avatar
Leo Gordon committed
460 461 462 463 464 465 466
    -help                  : print this help
    -dead                  : clean dead jobs for resubmission
    -alldead               : all outstanding workers
    -no_analysis_stats     : don't show status of each analysis
    -worker_stats          : show status of each running worker
    -failed_jobs           : show all failed jobs
    -reset_job_id <num>    : reset a job back to READY so it can be rerun
467 468
    -reset_failed_jobs_for_analysis <logic_name>
                           : reset FAILED jobs of an analysis back to READY so they can be rerun
Leo Gordon's avatar
Leo Gordon committed
469
    -reset_all_jobs_for_analysis <logic_name>
470
                           : reset ALL jobs of an analysis back to READY so they can be rerun
471

Leo Gordon's avatar
Leo Gordon committed
472 473
=head1 CONTACT

Leo Gordon's avatar
Leo Gordon committed
474
    Please contact ehive-users@ebi.ac.uk mailing list with questions/suggestions.
Leo Gordon's avatar
Leo Gordon committed
475

476 477
=cut