beekeeper.pl 23.6 KB
Newer Older
1
#!/usr/bin/env perl
2 3

use strict;
Leo Gordon's avatar
Leo Gordon committed
4
use warnings;
Leo Gordon's avatar
Leo Gordon committed
5

6 7 8 9
    # Finding out own path in order to reference own components (including own modules):
use Cwd            ();
use File::Basename ();
BEGIN {
10
    $ENV{'EHIVE_ROOT_DIR'} ||= File::Basename::dirname( File::Basename::dirname( Cwd::realpath($0) ) );
11
    unshift @INC, $ENV{'EHIVE_ROOT_DIR'}.'/modules';
12
}
13

14

15
use Getopt::Long;
16
use File::Path 'make_path';
17
use Bio::EnsEMBL::Hive::Utils ('script_usage', 'destringify', 'report_versions');
18
use Bio::EnsEMBL::Hive::Utils::Config;
19 20
use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
use Bio::EnsEMBL::Hive::Queen;
21
use Bio::EnsEMBL::Hive::Valley;
22
use Bio::EnsEMBL::Hive::Scheduler;
23

Leo Gordon's avatar
Leo Gordon committed
24 25 26 27 28 29 30 31 32 33 34
main();

sub main {
    $| = 1;

        # ok this is a hack, but I'm going to pretend I've got an object here
        # by creating a hash ref and passing it around like an object
        # this is to avoid using global variables in functions, and to consolidate
        # the globals into a nice '$self' package
    my $self = {};

35 36
    my $help                        = 0;
    my $report_versions             = 0;
Leo Gordon's avatar
Leo Gordon committed
37 38 39 40
    my $loopit                      = 0;
    my $sync                        = 0;
    my $local                       = 0;
    my $show_failed_jobs            = 0;
41
    my $default_meadow_type         = undef;
42 43 44
    my $submit_workers_max          = undef;
    my $total_running_workers_max   = undef;
    my $submission_options          = undef;
Leo Gordon's avatar
Leo Gordon committed
45
    my $run                         = 0;
46
    my $max_loops                   = 0; # not running by default
47
    my $run_job_id                  = undef;
48
    my $force                       = undef;
49
    my $keep_alive                  = 0; # ==1 means run even when there is nothing to do
Leo Gordon's avatar
Leo Gordon committed
50 51
    my $check_for_dead              = 0;
    my $all_dead                    = 0;
52
    my $balance_semaphores          = 0;
Leo Gordon's avatar
Leo Gordon committed
53 54 55 56 57
    my $job_id_for_output           = 0;
    my $show_worker_stats           = 0;
    my $kill_worker_id              = 0;
    my $reset_job_id                = 0;
    my $reset_all_jobs_for_analysis = 0;
58
    my $reset_failed_jobs_for_analysis = 0;
Leo Gordon's avatar
Leo Gordon committed
59

60
    $self->{'url'}                  = undef;
61
    $self->{'reg_conf'}             = undef;
62
    $self->{'reg_type'}             = undef;
63
    $self->{'reg_alias'}            = undef;
64
    $self->{'nosqlvc'}              = undef;
65

Leo Gordon's avatar
Leo Gordon committed
66
    $self->{'sleep_minutes'}        = 1;
67
    $self->{'retry_throwing_jobs'}  = undef;
68
    $self->{'can_respecialize'}     = undef;
69 70 71
    $self->{'hive_log_dir'}         = undef;
    $self->{'submit_stdout_file'}   = undef;
    $self->{'submit_stderr_file'}   = undef;
72
    $self->{'submit_log_dir'}       = undef;
Leo Gordon's avatar
Leo Gordon committed
73

Leo Gordon's avatar
Leo Gordon committed
74
    GetOptions(
Leo Gordon's avatar
Leo Gordon committed
75
                    # connection parameters
76
               'url=s'              => \$self->{'url'},
77
               'reg_conf|regfile=s' => \$self->{'reg_conf'},
78
               'reg_type=s'         => \$self->{'reg_type'},
79
               'reg_alias|regname=s'=> \$self->{'reg_alias'},
80
               'nosqlvc=i'          => \$self->{'nosqlvc'},     # can't use the binary "!" as it is a propagated option
Leo Gordon's avatar
Leo Gordon committed
81 82

                    # loop control
83 84 85 86
               'run'                => \$run,
               'loop'               => \$loopit,
               'max_loops=i'        => \$max_loops,
               'keep_alive'         => \$keep_alive,
87
               'job_id|run_job_id=i'=> \$run_job_id,
88
               'force=i'            => \$force,
89
               'sleep=f'            => \$self->{'sleep_minutes'},
Leo Gordon's avatar
Leo Gordon committed
90 91

                    # meadow control
92
               'local!'                         => \$local,
93
               'meadow_type=s'                  => \$default_meadow_type,
94 95 96
               'total_running_workers_max=i'    => \$total_running_workers_max,
               'submit_workers_max=i'           => \$submit_workers_max,
               'submission_options=s'           => \$submission_options,
Leo Gordon's avatar
Leo Gordon committed
97 98

                    # worker control
99
               'job_limit=i'            => \$self->{'job_limit'},
100
               'life_span|lifespan=i'   => \$self->{'life_span'},
101
               'logic_name=s'           => \$self->{'logic_name'},
102
               'hive_log_dir|hive_output_dir=s'      => \$self->{'hive_log_dir'},
103
               'retry_throwing_jobs=i'  => \$self->{'retry_throwing_jobs'},
104
               'can_respecialize=i'     => \$self->{'can_respecialize'},
105
               'debug=i'                => \$self->{'debug'},
106 107
               'submit_stdout_file=s'   => \$self->{'submit_stdout_file'},
               'submit_stderr_file=s'   => \$self->{'submit_stderr_file'},
108
               'submit_log_dir=s'       => \$self->{'submit_log_dir'},
Leo Gordon's avatar
Leo Gordon committed
109 110

                    # other commands/options
111 112 113 114
               'h|help!'           => \$help,
               'v|versions!'       => \$report_versions,
               'sync!'             => \$sync,
               'dead!'             => \$check_for_dead,
Leo Gordon's avatar
Leo Gordon committed
115
               'killworker=i'      => \$kill_worker_id,
116
               'alldead!'          => \$all_dead,
117
               'balance_semaphores'=> \$balance_semaphores,
Leo Gordon's avatar
Leo Gordon committed
118 119 120 121
               'no_analysis_stats' => \$self->{'no_analysis_stats'},
               'worker_stats'      => \$show_worker_stats,
               'failed_jobs'       => \$show_failed_jobs,
               'reset_job_id=i'    => \$reset_job_id,
122
               'reset_failed|reset_failed_jobs_for_analysis=s' => \$reset_failed_jobs_for_analysis,
Leo Gordon's avatar
Leo Gordon committed
123 124 125 126
               'reset_all|reset_all_jobs_for_analysis=s' => \$reset_all_jobs_for_analysis,
               'job_output=i'      => \$job_id_for_output,
    );

127
    if ($help) { script_usage(0); }
Leo Gordon's avatar
Leo Gordon committed
128

129 130 131 132 133
    if($report_versions) {
        report_versions();
        exit(0);
    }

134 135
    my $config = Bio::EnsEMBL::Hive::Utils::Config->new();      # will probably add a config_file option later

136
    if($run or $run_job_id) {
Leo Gordon's avatar
Leo Gordon committed
137
        $max_loops = 1;
138
    } elsif ($loopit or $keep_alive) {
Leo Gordon's avatar
Leo Gordon committed
139 140 141 142
        unless($max_loops) {
            $max_loops = -1; # unlimited
        }
    }
143

144 145 146 147 148 149 150 151
    if($self->{'url'} or $self->{'reg_alias'}) {
        $self->{'dba'} = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new(
            -url                            => $self->{'url'},
            -reg_conf                       => $self->{'reg_conf'},
            -reg_type                       => $self->{'reg_type'},
            -reg_alias                      => $self->{'reg_alias'},
            -no_sql_schema_version_check    => $self->{'nosqlvc'},
        );
Leo Gordon's avatar
Leo Gordon committed
152
    } else {
153
        print "\nERROR : Connection parameters (url or reg_conf+reg_alias) need to be specified\n\n";
154
        script_usage(1);
Leo Gordon's avatar
Leo Gordon committed
155
    }
156

157 158
    $self->{'safe_url'} = $self->{'dba'}->dbc->url('WORKER_PASSWORD');

Leo Gordon's avatar
Leo Gordon committed
159
    my $queen = $self->{'dba'}->get_Queen;
160

161
    my $pipeline_name = $self->{'dba'}->get_MetaAdaptor->get_value_by_key( 'hive_pipeline_name' );
162

163 164 165
    if($pipeline_name) {
        warn "Pipeline name: $pipeline_name\n";
    } else {
166 167 168 169 170 171 172 173 174 175 176 177 178
        print STDERR "+---------------------------------------------------------------------+\n";
        print STDERR "!                                                                     !\n";
        print STDERR "!                  WARNING:                                           !\n";
        print STDERR "!                                                                     !\n";
        print STDERR "! At the moment your pipeline doesn't have 'pipeline_name' defined.   !\n";
        print STDERR "! This may seriously impair your beekeeping experience unless you are !\n";
        print STDERR "! the only farm user. The name should be set in your PipeConfig file, !\n";
        print STDERR "! or if you are running an old pipeline you can just set it by hand   !\n";
        print STDERR "! in the 'meta' table.                                                !\n";
        print STDERR "!                                                                     !\n";
        print STDERR "+---------------------------------------------------------------------+\n";
    }

179 180 181 182
    if($run_job_id) {
        $submit_workers_max = 1;
    }

183 184
    $default_meadow_type = 'LOCAL' if($local);
    my $valley = Bio::EnsEMBL::Hive::Valley->new( $config, $default_meadow_type, $pipeline_name );
185 186 187 188 189 190 191

    my ($beekeeper_meadow_type, $beekeeper_meadow_name) = $valley->whereami();
    unless($beekeeper_meadow_type eq 'LOCAL') {
        die "beekeeper.pl detected it has been itself submitted to '$beekeeper_meadow_type/$beekeeper_meadow_name', but this mode of operation is not supported.\n"
           ."Please just run beekeeper.pl on a farm head node, preferably from under a 'screen' session.\n";
    }

192
    $valley->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
193

194 195
    my $default_meadow = $valley->get_default_meadow();
    warn "Default meadow: ".$default_meadow->signature."\n\n";
196

197 198
    $default_meadow->config_set('TotalRunningWorkersMax', $total_running_workers_max) if(defined $total_running_workers_max);
    $default_meadow->config_set('SubmissionOptions', $submission_options) if(defined $submission_options);
199

200
    if($reset_job_id) { $queen->reset_job_by_dbID_and_sync($reset_job_id); }
201

Leo Gordon's avatar
Leo Gordon committed
202 203 204
    if($job_id_for_output) {
        printf("===== job output\n");
        my $job = $self->{'dba'}->get_AnalysisJobAdaptor->fetch_by_dbID($job_id_for_output);
205
        print $job->toString. "\n";
Leo Gordon's avatar
Leo Gordon committed
206
    }
207

208
    if(my $reset_logic_name = $reset_all_jobs_for_analysis || $reset_failed_jobs_for_analysis) {
209

210 211
        my $reset_analysis = $self->{'dba'}->get_AnalysisAdaptor->fetch_by_logic_name($reset_logic_name)
              || die( "Cannot AnalysisAdaptor->fetch_by_logic_name($reset_logic_name)"); 
212

213 214
        $self->{'dba'}->get_AnalysisJobAdaptor->reset_jobs_for_analysis_id($reset_analysis->dbID, $reset_all_jobs_for_analysis); 
        $self->{'dba'}->get_Queen->synchronize_AnalysisStats($reset_analysis->stats);
Leo Gordon's avatar
Leo Gordon committed
215
    }
216

Leo Gordon's avatar
Leo Gordon committed
217
    if ($kill_worker_id) {
218
        my $kill_worker = $queen->fetch_by_dbID($kill_worker_id);
Javier Herrero's avatar
Javier Herrero committed
219

220 221
        unless( $kill_worker->cause_of_death() ) {
            if( my $meadow = $valley->find_available_meadow_responsible_for_worker( $kill_worker ) ) {
222

223 224
                if( $meadow->check_worker_is_alive_and_mine ) {
                    printf("Killing worker: %10d %35s %15s  %20s(%d) : ", 
225
                            $kill_worker->dbID, $kill_worker->host, $kill_worker->process_id, 
226
                            $kill_worker->analysis->logic_name, $kill_worker->analysis_id);
227

228 229 230
                    $meadow->kill_worker($kill_worker);
                    $kill_worker->cause_of_death('KILLED_BY_USER');
                    $queen->register_worker_death($kill_worker);
231 232 233 234
                         # what about clean-up? Should we do it here or not?
                } else {
                    die "According to the Meadow, the Worker (dbID=$kill_worker_id) is not running, so cannot kill";
                }
235
            } else {
236
                die "Cannot access the Meadow responsible for the Worker (dbID=$kill_worker_id), so cannot kill";
237 238
            }
        } else {
239
            die "According to the Queen, the Worker (dbID=$kill_worker_id) is not running, so cannot kill";
Leo Gordon's avatar
Leo Gordon committed
240 241
        }
    }
Javier Herrero's avatar
Javier Herrero committed
242

243 244 245
    my $analysis = $run_job_id
        ? $self->{'dba'}->get_AnalysisAdaptor->fetch_by_dbID( $self->{'dba'}->get_AnalysisJobAdaptor->fetch_by_dbID( $run_job_id )->analysis_id )
        : $self->{'dba'}->get_AnalysisAdaptor->fetch_by_logic_name($self->{'logic_name'});
246

247 248
    if($all_dead)           { $queen->register_all_workers_dead(); }
    if($check_for_dead)     { $queen->check_for_dead_workers($valley, 1); }
249
    if($balance_semaphores) { $self->{'dba'}->get_AnalysisJobAdaptor->balance_semaphores( $analysis && $analysis->dbID ); }
250

Leo Gordon's avatar
Leo Gordon committed
251
    if ($max_loops) { # positive $max_loop means limited, negative means unlimited
252

253
        run_autonomously($self, $max_loops, $keep_alive, $queen, $valley, $analysis, $run_job_id, $force);
Javier Herrero's avatar
Javier Herrero committed
254

Leo Gordon's avatar
Leo Gordon committed
255 256
    } else {
            # the output of several methods will look differently depending on $analysis being [un]defined
257

Leo Gordon's avatar
Leo Gordon committed
258 259 260 261
        if($sync) {
            $queen->synchronize_hive($analysis);
        }
        $queen->print_analysis_status($analysis) unless($self->{'no_analysis_stats'});
262

263
        if($show_worker_stats) {
Leo Gordon's avatar
Leo Gordon committed
264
            print "\n===== List of live Workers according to the Queen: ======\n";
265 266 267 268
            foreach my $worker (@{ $queen->fetch_overdue_workers(0) }) {
                print $worker->toString()."\n";
            }
        }
Leo Gordon's avatar
Leo Gordon committed
269
        $queen->print_running_worker_counts;
270

271
        Bio::EnsEMBL::Hive::Scheduler::schedule_workers_resync_if_necessary($queen, $valley, $analysis);   # show what would be submitted, but do not actually submit
Leo Gordon's avatar
Leo Gordon committed
272
        $queen->get_remaining_jobs_show_hive_progress();
273

Leo Gordon's avatar
Leo Gordon committed
274 275
        if($show_failed_jobs) {
            print("===== failed jobs\n");
276
            my $failed_job_list = $self->{'dba'}->get_AnalysisJobAdaptor->fetch_all_by_analysis_id_status($analysis && $analysis->dbID, 'FAILED');
277

Leo Gordon's avatar
Leo Gordon committed
278
            foreach my $job (@{$failed_job_list}) {
279
                print $job->toString. "\n";
Leo Gordon's avatar
Leo Gordon committed
280 281 282
            }
        }
    }
Javier Herrero's avatar
Javier Herrero committed
283

Leo Gordon's avatar
Leo Gordon committed
284
    exit(0);
285 286
}

287 288 289 290 291 292 293
#######################
#
# subroutines
#
#######################


Leo Gordon's avatar
Leo Gordon committed
294
sub generate_worker_cmd {
295
    my ($self, $run_analysis, $run_job_id, $force) = @_;
296

297
    my $worker_cmd = $ENV{'EHIVE_ROOT_DIR'}.'/scripts/runWorker.pl';
298 299 300

    unless(-x $worker_cmd) {
        print("Can't run '$worker_cmd' script for some reason, please investigate.\n");
301 302
        exit(1);
    }
303

304
    foreach my $worker_option ('url', 'reg_conf', 'reg_type', 'reg_alias', 'nosqlvc', 'job_limit', 'life_span', 'retry_throwing_jobs', 'can_respecialize', 'hive_log_dir', 'debug') {
305 306
        if(defined(my $value = $self->{$worker_option})) {
            $worker_cmd .= " -${worker_option} $value";
307
        }
Leo Gordon's avatar
Leo Gordon committed
308
    }
309

310
        # special task:
311
    if ($run_job_id) {
312
        $worker_cmd .= " -job_id $run_job_id";
313 314
    } elsif ($run_analysis) {
        $worker_cmd .= " -logic_name ".$run_analysis->logic_name;
315 316
    }

317 318 319 320
    if (defined($force)) {
        $worker_cmd .= " -force $force";
    }

Leo Gordon's avatar
Leo Gordon committed
321
    return $worker_cmd;
322 323 324
}

sub run_autonomously {
325
    my ($self, $max_loops, $keep_alive, $queen, $valley, $run_analysis, $run_job_id, $force) = @_;
326

327 328
    my $resourceless_worker_cmd = generate_worker_cmd($self, $run_analysis, $run_job_id, $force);
    my $special_task            = $run_analysis || $run_job_id;
Leo Gordon's avatar
Leo Gordon committed
329

330
    my $rc_id2name  = $self->{'dba'}->get_ResourceClassAdaptor->fetch_HASHED_FROM_resource_class_id_TO_name();
331
    my %meadow_type_rc_name2resource_param_list = ();
332
    foreach my $rd (@{ $self->{'dba'}->get_ResourceDescriptionAdaptor->fetch_all() }) {
333
        $meadow_type_rc_name2resource_param_list{ $rd->meadow_type() }{ $rc_id2name->{$rd->resource_class_id} } = [ $rd->submission_cmd_args, $rd->worker_cmd_args ];
334
    }
Leo Gordon's avatar
Leo Gordon committed
335

336 337
    my $beekeeper_pid = $$;

Leo Gordon's avatar
Leo Gordon committed
338 339 340 341 342 343 344 345
    my $iteration=0;
    my $num_of_remaining_jobs=0;
    my $failed_analyses=0;
    do {
        if($iteration++) {
            $self->{'dba'}->dbc->disconnect_if_idle;
            printf("sleep %.2f minutes. Next loop at %s\n", $self->{'sleep_minutes'}, scalar localtime(time+$self->{'sleep_minutes'}*60));
            sleep($self->{'sleep_minutes'}*60);  
346
        }
347

Leo Gordon's avatar
Leo Gordon committed
348
        print("\n======= beekeeper loop ** $iteration **==========\n");
Javier Herrero's avatar
Javier Herrero committed
349

350
        $queen->check_for_dead_workers($valley, 0);
351

Leo Gordon's avatar
Leo Gordon committed
352
        $queen->print_analysis_status unless($self->{'no_analysis_stats'});
Leo Gordon's avatar
Leo Gordon committed
353
        $queen->print_running_worker_counts;
Leo Gordon's avatar
Leo Gordon committed
354

355
        my $workers_to_submit_by_meadow_type_rc_name
356
            = Bio::EnsEMBL::Hive::Scheduler::schedule_workers_resync_if_necessary($queen, $valley, $run_analysis);
357

358
        if( keys %$workers_to_submit_by_meadow_type_rc_name ) {
359 360 361 362 363 364 365 366

            my $submit_log_subdir;

            if( $self->{'submit_log_dir'} ) {
                $submit_log_subdir = $self->{'submit_log_dir'}."/submit_bk${beekeeper_pid}_iter${iteration}";
                make_path( $submit_log_subdir );
            }

367
            foreach my $meadow_type (keys %$workers_to_submit_by_meadow_type_rc_name) {
Leo Gordon's avatar
Leo Gordon committed
368

369
                my $this_meadow = $valley->available_meadow_hash->{$meadow_type};
Leo Gordon's avatar
Leo Gordon committed
370

371 372 373 374 375
                foreach my $rc_name (keys %{ $workers_to_submit_by_meadow_type_rc_name->{$meadow_type} }) {
                    my $this_meadow_rc_worker_count = $workers_to_submit_by_meadow_type_rc_name->{$meadow_type}{$rc_name};

                    print "Submitting $this_meadow_rc_worker_count workers (rc_name=$rc_name) to ".$this_meadow->signature()."\n";

376 377 378
                    my ($submission_cmd_args, $worker_cmd_args) = @{ $meadow_type_rc_name2resource_param_list{ $meadow_type }{ $rc_name } || [] };

                    my $specific_worker_cmd = $resourceless_worker_cmd
379
                                            . " -rc_name $rc_name"
380 381
                                            . (defined($worker_cmd_args) ? " $worker_cmd_args" : '');

382
                    if( $self->{'submit_log_dir'} ) {
383 384
                        $self->{'submit_stdout_file'} = $submit_log_subdir . "/log_${rc_name}_%J_%I.out";
                        $self->{'submit_stderr_file'} = $submit_log_subdir . "/log_${rc_name}_%J_%I.err";
385 386
                    }

387 388
                    $this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration,
                                                    $rc_name, $submission_cmd_args || '',
389
                                                    $self->{'submit_stdout_file'}, $self->{'submit_stderr_file'});
390
                }
Leo Gordon's avatar
Leo Gordon committed
391
            }
Leo Gordon's avatar
Leo Gordon committed
392 393 394
        } else {
            print "Not submitting any workers this iteration\n";
        }
395

396
        $failed_analyses       = $queen->get_num_failed_analyses($run_analysis);
Leo Gordon's avatar
Leo Gordon committed
397
        $num_of_remaining_jobs = $queen->get_remaining_jobs_show_hive_progress();
398

399 400
    } while( $keep_alive
            or (!$failed_analyses and $num_of_remaining_jobs and $iteration!=$max_loops) );
401

Leo Gordon's avatar
Leo Gordon committed
402 403 404 405 406 407
    print "The Beekeeper has stopped because ".(
          $failed_analyses ? "there were $failed_analyses failed analyses"
        : !$num_of_remaining_jobs ? "there is nothing left to do"
        : "the number of loops was limited by $max_loops and this limit expired"
    )."\n";

Leo Gordon's avatar
Leo Gordon committed
408
    printf("dbc %d disconnect cycles\n", $self->{'dba'}->dbc->disconnect_count);
409 410
}

Leo Gordon's avatar
Leo Gordon committed
411

412 413 414 415 416 417 418 419 420 421
__DATA__

=pod

=head1 NAME

    beekeeper.pl

=head1 DESCRIPTION

Leo Gordon's avatar
Leo Gordon committed
422 423 424
    The Beekeeper is in charge of interfacing between the Queen and a compute resource or 'compute farm'.
    Its job is to initialize/sync the eHive database (via the Queen), query the Queen if it needs any workers
    and to send the requested number of workers to open machines via the runWorker.pl script.
Leo Gordon's avatar
Leo Gordon committed
425

Leo Gordon's avatar
Leo Gordon committed
426
    It is also responsible for interfacing with the Queen to identify workers which died
427
    unexpectedly so that she can free the dead workers and reclaim unfinished jobs.
428 429 430

=head1 USAGE EXAMPLES

Leo Gordon's avatar
Leo Gordon committed
431 432
        # Usually run after the pipeline has been created to calculate the internal statistics necessary for eHive functioning
    beekeeper.pl -url mysql://username:secret@hostname:port/ehive_dbname -sync
433

Leo Gordon's avatar
Leo Gordon committed
434 435
        # Do not run any additional Workers, just check for the current status of the pipeline:
    beekeeper.pl -url mysql://username:secret@hostname:port/ehive_dbname
Leo Gordon's avatar
Leo Gordon committed
436

437 438
        # Run the pipeline in automatic mode (-loop), run all the workers locally (-meadow_type LOCAL) and allow for 3 parallel workers (-total_running_workers_max 3)
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -meadow_type LOCAL -total_running_workers_max 3 -loop
439

Leo Gordon's avatar
Leo Gordon committed
440 441
        # Run in automatic mode, but only restrict to running the 'fast_blast' analysis
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -logic_name fast_blast -loop
442

Leo Gordon's avatar
Leo Gordon committed
443 444
        # Restrict the normal execution to one iteration only - can be used for testing a newly set up pipeline
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -run
445

446 447
        # Reset failed 'buggy_analysis' jobs to 'READY' state, so that they can be run again
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -reset_failed_jobs_for_analysis buggy_analysis
448

Leo Gordon's avatar
Leo Gordon committed
449 450
        # Do a cleanup: find and bury dead workers, reclaim their jobs
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -dead
451 452 453 454 455

=head1 OPTIONS

=head2 Connection parameters

456
    -reg_conf <path>       : path to a Registry configuration file
457
    -reg_type <string>     : type of the registry entry ('hive', 'core', 'compara', etc - defaults to 'hive')
458
    -reg_alias <string>    : species/alias name for the Hive DBAdaptor
Leo Gordon's avatar
Leo Gordon committed
459
    -url <url string>      : url defining where hive database is located
460 461 462

=head2 Looping control

Leo Gordon's avatar
Leo Gordon committed
463 464
    -loop                  : run autonomously, loops and sleeps
    -max_loops <num>       : perform max this # of loops in autonomous mode
465
    -keep_alive            : do not stop when there are no more jobs to do - carry on looping
Leo Gordon's avatar
Leo Gordon committed
466
    -job_id <job_id>       : run 1 iteration for this job_id
467
    -run                   : run 1 iteration of automation loop
Leo Gordon's avatar
Leo Gordon committed
468
    -sleep <num>           : when looping, sleep <num> minutes (default 2min)
469

470
=head2 Current Meadow control
471

472 473 474 475
    -meadow_type <string>               : the desired Meadow class name, such as 'LSF' or 'LOCAL'
    -total_running_workers_max <num>    : max # workers to be running in parallel
    -submit_workers_max <num>           : max # workers to create per loop iteration
    -submission_options <string>        : passes <string> to the Meadow submission command as <options> (formerly lsf_options)
476
    -submit_log_dir <dir>               : record submission output+error streams into files under the given directory (to see why some workers fail after submission)
477 478 479

=head2 Worker control

480
    -job_limit <num>            : #jobs to run before worker can die naturally
481
    -life_span <num>            : life_span limit for each worker
482 483
    -logic_name <string>        : restrict the pipeline stat/runs to this analysis logic_name
    -retry_throwing_jobs 0|1    : if a job dies *knowingly*, should we retry it by default?
484
    -can_respecialize <0|1>     : allow workers to re-specialize into another analysis (within resource_class) after their previous analysis was exhausted
485
    -hive_log_dir <path>        : directory where stdout/stderr of the hive is redirected
486
    -debug <debug_level>        : set debug level of the workers
487 488 489

=head2 Other commands/options

Leo Gordon's avatar
Leo Gordon committed
490
    -help                  : print this help
491
    -versions              : report both Hive code version and Hive database schema version
492 493 494
    -dead                  : detect all unaccounted dead workers and reset their jobs for resubmission
    -alldead               : tell the database all workers are dead (no checks are performed in this mode, so be very careful!)
    -balance_semaphores    : set all semaphore_counts to the numbers of unDONE fan jobs (emergency use only)
Leo Gordon's avatar
Leo Gordon committed
495 496 497 498
    -no_analysis_stats     : don't show status of each analysis
    -worker_stats          : show status of each running worker
    -failed_jobs           : show all failed jobs
    -reset_job_id <num>    : reset a job back to READY so it can be rerun
499 500
    -reset_failed_jobs_for_analysis <logic_name>
                           : reset FAILED jobs of an analysis back to READY so they can be rerun
Leo Gordon's avatar
Leo Gordon committed
501
    -reset_all_jobs_for_analysis <logic_name>
502
                           : reset ALL jobs of an analysis back to READY so they can be rerun
503

504 505
=head1 LICENSE

506
    Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
507 508 509 510 511 512 513 514 515 516

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software distributed under the License
    is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and limitations under the License.

Leo Gordon's avatar
Leo Gordon committed
517 518
=head1 CONTACT

519
    Please subscribe to the Hive mailing list:  http://listserver.ebi.ac.uk/mailman/listinfo/ehive-users  to discuss Hive-related questions or to be notified of our updates
Leo Gordon's avatar
Leo Gordon committed
520

521 522
=cut