beekeeper.pl 24 KB
Newer Older
1
#!/usr/bin/env perl
2 3

use strict;
Leo Gordon's avatar
Leo Gordon committed
4
use warnings;
Leo Gordon's avatar
Leo Gordon committed
5

6 7 8 9
    # Finding out own path in order to reference own components (including own modules):
use Cwd            ();
use File::Basename ();
BEGIN {
10
    $ENV{'EHIVE_ROOT_DIR'} ||= File::Basename::dirname( File::Basename::dirname( Cwd::realpath($0) ) );
11
    unshift @INC, $ENV{'EHIVE_ROOT_DIR'}.'/modules';
12
}
13

14

15
use Getopt::Long;
16
use File::Path 'make_path';
17
use Bio::EnsEMBL::Hive::Utils ('script_usage', 'destringify', 'report_versions');
18
use Bio::EnsEMBL::Hive::Utils::Config;
19 20
use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
use Bio::EnsEMBL::Hive::Queen;
21
use Bio::EnsEMBL::Hive::Valley;
22
use Bio::EnsEMBL::Hive::Scheduler;
23

Leo Gordon's avatar
Leo Gordon committed
24 25 26 27 28 29 30 31 32 33 34 35
main();

sub main {
    $| = 1;
    Bio::EnsEMBL::Registry->no_version_check(1);

        # ok this is a hack, but I'm going to pretend I've got an object here
        # by creating a hash ref and passing it around like an object
        # this is to avoid using global variables in functions, and to consolidate
        # the globals into a nice '$self' package
    my $self = {};

36 37
    my $help                        = 0;
    my $report_versions             = 0;
Leo Gordon's avatar
Leo Gordon committed
38 39 40 41
    my $loopit                      = 0;
    my $sync                        = 0;
    my $local                       = 0;
    my $show_failed_jobs            = 0;
42
    my $default_meadow_type         = undef;
43 44 45
    my $submit_workers_max          = undef;
    my $total_running_workers_max   = undef;
    my $submission_options          = undef;
Leo Gordon's avatar
Leo Gordon committed
46
    my $run                         = 0;
47
    my $max_loops                   = 0; # not running by default
48
    my $run_job_id                  = undef;
49
    my $force                       = undef;
50
    my $keep_alive                  = 0; # ==1 means run even when there is nothing to do
Leo Gordon's avatar
Leo Gordon committed
51 52
    my $check_for_dead              = 0;
    my $all_dead                    = 0;
53
    my $balance_semaphores          = 0;
Leo Gordon's avatar
Leo Gordon committed
54 55 56 57 58
    my $job_id_for_output           = 0;
    my $show_worker_stats           = 0;
    my $kill_worker_id              = 0;
    my $reset_job_id                = 0;
    my $reset_all_jobs_for_analysis = 0;
59
    my $reset_failed_jobs_for_analysis = 0;
Leo Gordon's avatar
Leo Gordon committed
60

61
    $self->{'url'}                  = undef;
62
    $self->{'reg_conf'}             = undef;
63
    $self->{'reg_type'}             = undef;
64
    $self->{'reg_alias'}            = undef;
65
    $self->{'nosqlvc'}              = undef;
66

Leo Gordon's avatar
Leo Gordon committed
67
    $self->{'sleep_minutes'}        = 1;
68
    $self->{'retry_throwing_jobs'}  = undef;
69
    $self->{'can_respecialize'}     = undef;
70 71 72
    $self->{'hive_log_dir'}         = undef;
    $self->{'submit_stdout_file'}   = undef;
    $self->{'submit_stderr_file'}   = undef;
73
    $self->{'submit_log_dir'}       = undef;
Leo Gordon's avatar
Leo Gordon committed
74

Leo Gordon's avatar
Leo Gordon committed
75
    GetOptions(
Leo Gordon's avatar
Leo Gordon committed
76
                    # connection parameters
77
               'url=s'              => \$self->{'url'},
78
               'reg_conf|regfile=s' => \$self->{'reg_conf'},
79
               'reg_type=s'         => \$self->{'reg_type'},
80
               'reg_alias|regname=s'=> \$self->{'reg_alias'},
81
               'nosqlvc=i'          => \$self->{'nosqlvc'},     # can't use the binary "!" as it is a propagated option
Leo Gordon's avatar
Leo Gordon committed
82 83

                    # loop control
84 85 86 87
               'run'                => \$run,
               'loop'               => \$loopit,
               'max_loops=i'        => \$max_loops,
               'keep_alive'         => \$keep_alive,
88
               'job_id|run_job_id=i'=> \$run_job_id,
89
               'force=i'            => \$force,
90
               'sleep=f'            => \$self->{'sleep_minutes'},
Leo Gordon's avatar
Leo Gordon committed
91 92

                    # meadow control
93
               'local!'                         => \$local,
94
               'meadow_type=s'                  => \$default_meadow_type,
95 96 97
               'total_running_workers_max=i'    => \$total_running_workers_max,
               'submit_workers_max=i'           => \$submit_workers_max,
               'submission_options=s'           => \$submission_options,
Leo Gordon's avatar
Leo Gordon committed
98 99

                    # worker control
100
               'job_limit=i'            => \$self->{'job_limit'},
101
               'life_span|lifespan=i'   => \$self->{'life_span'},
102
               'logic_name=s'           => \$self->{'logic_name'},
103
               'hive_log_dir|hive_output_dir=s'      => \$self->{'hive_log_dir'},
104
               'retry_throwing_jobs=i'  => \$self->{'retry_throwing_jobs'},
105
               'can_respecialize=i'     => \$self->{'can_respecialize'},
106
               'debug=i'                => \$self->{'debug'},
107 108
               'submit_stdout_file=s'   => \$self->{'submit_stdout_file'},
               'submit_stderr_file=s'   => \$self->{'submit_stderr_file'},
109
               'submit_log_dir=s'       => \$self->{'submit_log_dir'},
Leo Gordon's avatar
Leo Gordon committed
110 111

                    # other commands/options
112 113 114 115
               'h|help!'           => \$help,
               'v|versions!'       => \$report_versions,
               'sync!'             => \$sync,
               'dead!'             => \$check_for_dead,
Leo Gordon's avatar
Leo Gordon committed
116
               'killworker=i'      => \$kill_worker_id,
117
               'alldead!'          => \$all_dead,
118
               'balance_semaphores'=> \$balance_semaphores,
Leo Gordon's avatar
Leo Gordon committed
119 120 121 122
               'no_analysis_stats' => \$self->{'no_analysis_stats'},
               'worker_stats'      => \$show_worker_stats,
               'failed_jobs'       => \$show_failed_jobs,
               'reset_job_id=i'    => \$reset_job_id,
123
               'reset_failed|reset_failed_jobs_for_analysis=s' => \$reset_failed_jobs_for_analysis,
Leo Gordon's avatar
Leo Gordon committed
124 125 126 127 128
               'reset_all|reset_all_jobs_for_analysis=s' => \$reset_all_jobs_for_analysis,
               'job_output=i'      => \$job_id_for_output,
               'monitor!'          => \$self->{'monitor'},
    );

129
    if ($help) { script_usage(0); }
Leo Gordon's avatar
Leo Gordon committed
130

131 132 133 134 135
    if($report_versions) {
        report_versions();
        exit(0);
    }

136 137
    my $config = Bio::EnsEMBL::Hive::Utils::Config->new();      # will probably add a config_file option later

138
    if($run or $run_job_id) {
Leo Gordon's avatar
Leo Gordon committed
139
        $max_loops = 1;
140
    } elsif ($loopit or $keep_alive) {
Leo Gordon's avatar
Leo Gordon committed
141 142 143 144 145 146 147
        unless($max_loops) {
            $max_loops = -1; # unlimited
        }
        unless(defined($self->{'monitor'})) {
            $self->{'monitor'} = 1;
        }
    }
148

149 150 151 152 153 154 155 156
    if($self->{'url'} or $self->{'reg_alias'}) {
        $self->{'dba'} = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new(
            -url                            => $self->{'url'},
            -reg_conf                       => $self->{'reg_conf'},
            -reg_type                       => $self->{'reg_type'},
            -reg_alias                      => $self->{'reg_alias'},
            -no_sql_schema_version_check    => $self->{'nosqlvc'},
        );
Leo Gordon's avatar
Leo Gordon committed
157
    } else {
158
        print "\nERROR : Connection parameters (url or reg_conf+reg_alias) need to be specified\n\n";
159
        script_usage(1);
Leo Gordon's avatar
Leo Gordon committed
160
    }
161

162 163 164
    if( $self->{'url'} ) {    # protect the URL that we pass to Workers by hiding the password in %ENV:
        $self->{'url'} = $self->{'dba'}->dbc->url('EHIVE_PASS');
    }
165

Leo Gordon's avatar
Leo Gordon committed
166
    my $queen = $self->{'dba'}->get_Queen;
167

168
    my $pipeline_name = $self->{'dba'}->get_MetaAdaptor->fetch_value_by_key( 'hive_pipeline_name' );
169

170 171 172
    if($pipeline_name) {
        warn "Pipeline name: $pipeline_name\n";
    } else {
173 174 175 176 177 178 179 180 181 182 183 184 185
        print STDERR "+---------------------------------------------------------------------+\n";
        print STDERR "!                                                                     !\n";
        print STDERR "!                  WARNING:                                           !\n";
        print STDERR "!                                                                     !\n";
        print STDERR "! At the moment your pipeline doesn't have 'pipeline_name' defined.   !\n";
        print STDERR "! This may seriously impair your beekeeping experience unless you are !\n";
        print STDERR "! the only farm user. The name should be set in your PipeConfig file, !\n";
        print STDERR "! or if you are running an old pipeline you can just set it by hand   !\n";
        print STDERR "! in the 'meta' table.                                                !\n";
        print STDERR "!                                                                     !\n";
        print STDERR "+---------------------------------------------------------------------+\n";
    }

186 187 188 189
    if($run_job_id) {
        $submit_workers_max = 1;
    }

190 191
    $default_meadow_type = 'LOCAL' if($local);
    my $valley = Bio::EnsEMBL::Hive::Valley->new( $config, $default_meadow_type, $pipeline_name );
192 193 194 195 196 197 198

    my ($beekeeper_meadow_type, $beekeeper_meadow_name) = $valley->whereami();
    unless($beekeeper_meadow_type eq 'LOCAL') {
        die "beekeeper.pl detected it has been itself submitted to '$beekeeper_meadow_type/$beekeeper_meadow_name', but this mode of operation is not supported.\n"
           ."Please just run beekeeper.pl on a farm head node, preferably from under a 'screen' session.\n";
    }

199
    $valley->config_set('SubmitWorkersMax', $submit_workers_max) if(defined $submit_workers_max);
200

201 202
    my $default_meadow = $valley->get_default_meadow();
    warn "Default meadow: ".$default_meadow->signature."\n\n";
203

204 205
    $default_meadow->config_set('TotalRunningWorkersMax', $total_running_workers_max) if(defined $total_running_workers_max);
    $default_meadow->config_set('SubmissionOptions', $submission_options) if(defined $submission_options);
206

207
    if($reset_job_id) { $queen->reset_job_by_dbID_and_sync($reset_job_id); }
208

Leo Gordon's avatar
Leo Gordon committed
209 210 211 212 213
    if($job_id_for_output) {
        printf("===== job output\n");
        my $job = $self->{'dba'}->get_AnalysisJobAdaptor->fetch_by_dbID($job_id_for_output);
        $job->print_job();
    }
214

215
    if(my $reset_logic_name = $reset_all_jobs_for_analysis || $reset_failed_jobs_for_analysis) {
216

217 218
        my $reset_analysis = $self->{'dba'}->get_AnalysisAdaptor->fetch_by_logic_name($reset_logic_name)
              || die( "Cannot AnalysisAdaptor->fetch_by_logic_name($reset_logic_name)"); 
219

220 221
        $self->{'dba'}->get_AnalysisJobAdaptor->reset_jobs_for_analysis_id($reset_analysis->dbID, $reset_all_jobs_for_analysis); 
        $self->{'dba'}->get_Queen->synchronize_AnalysisStats($reset_analysis->stats);
Leo Gordon's avatar
Leo Gordon committed
222
    }
223

Leo Gordon's avatar
Leo Gordon committed
224
    if ($kill_worker_id) {
225
        my $kill_worker = $queen->fetch_by_dbID($kill_worker_id);
Javier Herrero's avatar
Javier Herrero committed
226

227 228
        unless( $kill_worker->cause_of_death() ) {
            if( my $meadow = $valley->find_available_meadow_responsible_for_worker( $kill_worker ) ) {
229

230
                if( $meadow->check_worker_is_alive_and_mine($kill_worker) ) {
231
                    printf("Killing worker: %10d %35s %15s  %20s(%d) : ", 
232
                            $kill_worker->dbID, $kill_worker->host, $kill_worker->process_id, 
233
                            $kill_worker->analysis->logic_name, $kill_worker->analysis_id);
234

235 236 237
                    $meadow->kill_worker($kill_worker);
                    $kill_worker->cause_of_death('KILLED_BY_USER');
                    $queen->register_worker_death($kill_worker);
238 239 240 241
                         # what about clean-up? Should we do it here or not?
                } else {
                    die "According to the Meadow, the Worker (dbID=$kill_worker_id) is not running, so cannot kill";
                }
242
            } else {
243
                die "Cannot access the Meadow responsible for the Worker (dbID=$kill_worker_id), so cannot kill";
244 245
            }
        } else {
246
            die "According to the Queen, the Worker (dbID=$kill_worker_id) is not running, so cannot kill";
Leo Gordon's avatar
Leo Gordon committed
247 248
        }
    }
Javier Herrero's avatar
Javier Herrero committed
249

250 251 252
    my $analysis = $run_job_id
        ? $self->{'dba'}->get_AnalysisAdaptor->fetch_by_dbID( $self->{'dba'}->get_AnalysisJobAdaptor->fetch_by_dbID( $run_job_id )->analysis_id )
        : $self->{'dba'}->get_AnalysisAdaptor->fetch_by_logic_name($self->{'logic_name'});
253

254 255
    if($all_dead)           { $queen->register_all_workers_dead(); }
    if($check_for_dead)     { $queen->check_for_dead_workers($valley, 1); }
256
    if($balance_semaphores) { $self->{'dba'}->get_AnalysisJobAdaptor->balance_semaphores( $analysis && $analysis->dbID ); }
257

Leo Gordon's avatar
Leo Gordon committed
258
    if ($max_loops) { # positive $max_loop means limited, negative means unlimited
259

260
        run_autonomously($self, $max_loops, $keep_alive, $queen, $valley, $analysis, $run_job_id, $force);
Javier Herrero's avatar
Javier Herrero committed
261

Leo Gordon's avatar
Leo Gordon committed
262 263
    } else {
            # the output of several methods will look differently depending on $analysis being [un]defined
264

Leo Gordon's avatar
Leo Gordon committed
265 266 267 268
        if($sync) {
            $queen->synchronize_hive($analysis);
        }
        $queen->print_analysis_status($analysis) unless($self->{'no_analysis_stats'});
269

270
        if($show_worker_stats) {
Leo Gordon's avatar
Leo Gordon committed
271
            print "\n===== List of live Workers according to the Queen: ======\n";
272 273 274 275
            foreach my $worker (@{ $queen->fetch_overdue_workers(0) }) {
                print $worker->toString()."\n";
            }
        }
Leo Gordon's avatar
Leo Gordon committed
276
        $queen->print_running_worker_counts;
277

278
        Bio::EnsEMBL::Hive::Scheduler::schedule_workers_resync_if_necessary($queen, $valley, $analysis);   # show what would be submitted, but do not actually submit
Leo Gordon's avatar
Leo Gordon committed
279
        $queen->get_remaining_jobs_show_hive_progress();
280

Leo Gordon's avatar
Leo Gordon committed
281 282
        if($show_failed_jobs) {
            print("===== failed jobs\n");
283
            my $failed_job_list = $self->{'dba'}->get_AnalysisJobAdaptor->fetch_all_by_analysis_id_status($analysis && $analysis->dbID, 'FAILED');
284

Leo Gordon's avatar
Leo Gordon committed
285 286 287 288 289
            foreach my $job (@{$failed_job_list}) {
                $job->print_job();
            }
        }
    }
Javier Herrero's avatar
Javier Herrero committed
290

Leo Gordon's avatar
Leo Gordon committed
291 292 293
    if ($self->{'monitor'}) {
        $queen->monitor();
    }
294

Leo Gordon's avatar
Leo Gordon committed
295
    exit(0);
296 297
}

298 299 300 301 302 303 304
#######################
#
# subroutines
#
#######################


Leo Gordon's avatar
Leo Gordon committed
305
sub generate_worker_cmd {
306
    my ($self, $run_analysis, $run_job_id, $force) = @_;
307

308
    my $worker_cmd = $ENV{'EHIVE_ROOT_DIR'}.'/scripts/runWorker.pl';
309 310 311

    unless(-x $worker_cmd) {
        print("Can't run '$worker_cmd' script for some reason, please investigate.\n");
312 313
        exit(1);
    }
314

315
    foreach my $worker_option ('url', 'reg_conf', 'reg_type', 'reg_alias', 'nosqlvc', 'job_limit', 'life_span', 'retry_throwing_jobs', 'can_respecialize', 'hive_log_dir', 'debug') {
316 317
        if(defined(my $value = $self->{$worker_option})) {
            $worker_cmd .= " -${worker_option} $value";
318
        }
Leo Gordon's avatar
Leo Gordon committed
319
    }
320

321
        # special task:
322
    if ($run_job_id) {
323
        $worker_cmd .= " -job_id $run_job_id";
324 325
    } elsif ($run_analysis) {
        $worker_cmd .= " -logic_name ".$run_analysis->logic_name;
326 327
    }

328 329 330 331
    if (defined($force)) {
        $worker_cmd .= " -force $force";
    }

Leo Gordon's avatar
Leo Gordon committed
332
    return $worker_cmd;
333 334 335
}

sub run_autonomously {
336
    my ($self, $max_loops, $keep_alive, $queen, $valley, $run_analysis, $run_job_id, $force) = @_;
337

338 339
    my $resourceless_worker_cmd = generate_worker_cmd($self, $run_analysis, $run_job_id, $force);
    my $special_task            = $run_analysis || $run_job_id;
Leo Gordon's avatar
Leo Gordon committed
340

341
    my $rc_id2name  = $self->{'dba'}->get_ResourceClassAdaptor->fetch_HASHED_FROM_resource_class_id_TO_name();
342
    my %meadow_type_rc_name2resource_param_list = ();
343
    foreach my $rd (@{ $self->{'dba'}->get_ResourceDescriptionAdaptor->fetch_all() }) {
344
        $meadow_type_rc_name2resource_param_list{ $rd->meadow_type() }{ $rc_id2name->{$rd->resource_class_id} } = [ $rd->submission_cmd_args, $rd->worker_cmd_args ];
345
    }
Leo Gordon's avatar
Leo Gordon committed
346

347 348
    my $beekeeper_pid = $$;

Leo Gordon's avatar
Leo Gordon committed
349 350 351 352 353 354 355 356 357
    my $iteration=0;
    my $num_of_remaining_jobs=0;
    my $failed_analyses=0;
    do {
        if($iteration++) {
            $queen->monitor();
            $self->{'dba'}->dbc->disconnect_if_idle;
            printf("sleep %.2f minutes. Next loop at %s\n", $self->{'sleep_minutes'}, scalar localtime(time+$self->{'sleep_minutes'}*60));
            sleep($self->{'sleep_minutes'}*60);  
358
        }
359

Leo Gordon's avatar
Leo Gordon committed
360
        print("\n======= beekeeper loop ** $iteration **==========\n");
Javier Herrero's avatar
Javier Herrero committed
361

362
        $queen->check_for_dead_workers($valley, 0);
363

Leo Gordon's avatar
Leo Gordon committed
364
        $queen->print_analysis_status unless($self->{'no_analysis_stats'});
Leo Gordon's avatar
Leo Gordon committed
365
        $queen->print_running_worker_counts;
Leo Gordon's avatar
Leo Gordon committed
366

367
        my $workers_to_submit_by_meadow_type_rc_name
368
            = Bio::EnsEMBL::Hive::Scheduler::schedule_workers_resync_if_necessary($queen, $valley, $run_analysis);
369

370
        if( keys %$workers_to_submit_by_meadow_type_rc_name ) {
371 372 373 374 375 376 377 378

            my $submit_log_subdir;

            if( $self->{'submit_log_dir'} ) {
                $submit_log_subdir = $self->{'submit_log_dir'}."/submit_bk${beekeeper_pid}_iter${iteration}";
                make_path( $submit_log_subdir );
            }

379
            foreach my $meadow_type (keys %$workers_to_submit_by_meadow_type_rc_name) {
Leo Gordon's avatar
Leo Gordon committed
380

381
                my $this_meadow = $valley->available_meadow_hash->{$meadow_type};
Leo Gordon's avatar
Leo Gordon committed
382

383 384 385 386 387
                foreach my $rc_name (keys %{ $workers_to_submit_by_meadow_type_rc_name->{$meadow_type} }) {
                    my $this_meadow_rc_worker_count = $workers_to_submit_by_meadow_type_rc_name->{$meadow_type}{$rc_name};

                    print "Submitting $this_meadow_rc_worker_count workers (rc_name=$rc_name) to ".$this_meadow->signature()."\n";

388 389 390 391 392 393
                    my ($submission_cmd_args, $worker_cmd_args) = @{ $meadow_type_rc_name2resource_param_list{ $meadow_type }{ $rc_name } || [] };

                    my $specific_worker_cmd = $resourceless_worker_cmd
                                            . ($special_task ? '' : " -rc_name $rc_name")
                                            . (defined($worker_cmd_args) ? " $worker_cmd_args" : '');

394
                    if( $self->{'submit_log_dir'} ) {
395 396
                        $self->{'submit_stdout_file'} = $submit_log_subdir . "/log_${rc_name}_%J_%I.out";
                        $self->{'submit_stderr_file'} = $submit_log_subdir . "/log_${rc_name}_%J_%I.err";
397 398
                    }

399 400
                    $this_meadow->submit_workers($specific_worker_cmd, $this_meadow_rc_worker_count, $iteration,
                                                    $rc_name, $submission_cmd_args || '',
401
                                                    $self->{'submit_stdout_file'}, $self->{'submit_stderr_file'});
402
                }
Leo Gordon's avatar
Leo Gordon committed
403
            }
Leo Gordon's avatar
Leo Gordon committed
404 405 406
        } else {
            print "Not submitting any workers this iteration\n";
        }
407

408
        $failed_analyses       = $queen->get_num_failed_analyses($run_analysis);
Leo Gordon's avatar
Leo Gordon committed
409
        $num_of_remaining_jobs = $queen->get_remaining_jobs_show_hive_progress();
410

411 412
    } while( $keep_alive
            or (!$failed_analyses and $num_of_remaining_jobs and $iteration!=$max_loops) );
413

Leo Gordon's avatar
Leo Gordon committed
414 415 416 417 418 419
    print "The Beekeeper has stopped because ".(
          $failed_analyses ? "there were $failed_analyses failed analyses"
        : !$num_of_remaining_jobs ? "there is nothing left to do"
        : "the number of loops was limited by $max_loops and this limit expired"
    )."\n";

Leo Gordon's avatar
Leo Gordon committed
420
    printf("dbc %d disconnect cycles\n", $self->{'dba'}->dbc->disconnect_count);
421 422
}

Leo Gordon's avatar
Leo Gordon committed
423

424 425 426 427 428 429 430 431 432 433
__DATA__

=pod

=head1 NAME

    beekeeper.pl

=head1 DESCRIPTION

Leo Gordon's avatar
Leo Gordon committed
434 435 436
    The Beekeeper is in charge of interfacing between the Queen and a compute resource or 'compute farm'.
    Its job is to initialize/sync the eHive database (via the Queen), query the Queen if it needs any workers
    and to send the requested number of workers to open machines via the runWorker.pl script.
Leo Gordon's avatar
Leo Gordon committed
437

Leo Gordon's avatar
Leo Gordon committed
438
    It is also responsible for interfacing with the Queen to identify workers which died
439
    unexpectedly so that she can free the dead workers and reclaim unfinished jobs.
440 441 442

=head1 USAGE EXAMPLES

Leo Gordon's avatar
Leo Gordon committed
443 444
        # Usually run after the pipeline has been created to calculate the internal statistics necessary for eHive functioning
    beekeeper.pl -url mysql://username:secret@hostname:port/ehive_dbname -sync
445

Leo Gordon's avatar
Leo Gordon committed
446 447
        # Do not run any additional Workers, just check for the current status of the pipeline:
    beekeeper.pl -url mysql://username:secret@hostname:port/ehive_dbname
Leo Gordon's avatar
Leo Gordon committed
448

449 450
        # Run the pipeline in automatic mode (-loop), run all the workers locally (-meadow_type LOCAL) and allow for 3 parallel workers (-total_running_workers_max 3)
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -meadow_type LOCAL -total_running_workers_max 3 -loop
451

Leo Gordon's avatar
Leo Gordon committed
452 453
        # Run in automatic mode, but only restrict to running the 'fast_blast' analysis
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -logic_name fast_blast -loop
454

Leo Gordon's avatar
Leo Gordon committed
455 456
        # Restrict the normal execution to one iteration only - can be used for testing a newly set up pipeline
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -run
457

458 459
        # Reset failed 'buggy_analysis' jobs to 'READY' state, so that they can be run again
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -reset_failed_jobs_for_analysis buggy_analysis
460

Leo Gordon's avatar
Leo Gordon committed
461 462
        # Do a cleanup: find and bury dead workers, reclaim their jobs
    beekeeper.pl -url mysql://username:secret@hostname:port/long_mult_test -dead
463 464 465 466 467

=head1 OPTIONS

=head2 Connection parameters

468
    -reg_conf <path>       : path to a Registry configuration file
469
    -reg_type <string>     : type of the registry entry ('hive', 'core', 'compara', etc - defaults to 'hive')
470
    -reg_alias <string>    : species/alias name for the Hive DBAdaptor
Leo Gordon's avatar
Leo Gordon committed
471
    -url <url string>      : url defining where hive database is located
472 473 474

=head2 Looping control

Leo Gordon's avatar
Leo Gordon committed
475 476
    -loop                  : run autonomously, loops and sleeps
    -max_loops <num>       : perform max this # of loops in autonomous mode
477
    -keep_alive            : do not stop when there are no more jobs to do - carry on looping
Leo Gordon's avatar
Leo Gordon committed
478
    -job_id <job_id>       : run 1 iteration for this job_id
479
    -run                   : run 1 iteration of automation loop
Leo Gordon's avatar
Leo Gordon committed
480
    -sleep <num>           : when looping, sleep <num> minutes (default 2min)
481

482
=head2 Current Meadow control
483

484 485 486 487
    -meadow_type <string>               : the desired Meadow class name, such as 'LSF' or 'LOCAL'
    -total_running_workers_max <num>    : max # workers to be running in parallel
    -submit_workers_max <num>           : max # workers to create per loop iteration
    -submission_options <string>        : passes <string> to the Meadow submission command as <options> (formerly lsf_options)
488
    -submit_log_dir <dir>               : record submission output+error streams into files under the given directory (to see why some workers fail after submission)
489 490 491

=head2 Worker control

492
    -job_limit <num>            : #jobs to run before worker can die naturally
493
    -life_span <num>            : number of minutes each worker is allowed to run
494 495
    -logic_name <string>        : restrict the pipeline stat/runs to this analysis logic_name
    -retry_throwing_jobs 0|1    : if a job dies *knowingly*, should we retry it by default?
496
    -can_respecialize <0|1>     : allow workers to re-specialize into another analysis (within resource_class) after their previous analysis was exhausted
497
    -hive_log_dir <path>        : directory where stdout/stderr of the hive is redirected
498
    -debug <debug_level>        : set debug level of the workers
499 500 501

=head2 Other commands/options

Leo Gordon's avatar
Leo Gordon committed
502
    -help                  : print this help
503
    -versions              : report both Hive code version and Hive database schema version
504 505 506
    -dead                  : detect all unaccounted dead workers and reset their jobs for resubmission
    -alldead               : tell the database all workers are dead (no checks are performed in this mode, so be very careful!)
    -balance_semaphores    : set all semaphore_counts to the numbers of unDONE fan jobs (emergency use only)
Leo Gordon's avatar
Leo Gordon committed
507 508 509 510
    -no_analysis_stats     : don't show status of each analysis
    -worker_stats          : show status of each running worker
    -failed_jobs           : show all failed jobs
    -reset_job_id <num>    : reset a job back to READY so it can be rerun
511 512
    -reset_failed_jobs_for_analysis <logic_name>
                           : reset FAILED jobs of an analysis back to READY so they can be rerun
Leo Gordon's avatar
Leo Gordon committed
513
    -reset_all_jobs_for_analysis <logic_name>
514
                           : reset ALL jobs of an analysis back to READY so they can be rerun
515

516 517
=head1 LICENSE

518
    Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Matthieu Muffato's avatar
Matthieu Muffato committed
519
    Copyright [2016-2018] EMBL-European Bioinformatics Institute
520 521 522 523 524 525 526 527 528 529

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software distributed under the License
    is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and limitations under the License.

Leo Gordon's avatar
Leo Gordon committed
530 531
=head1 CONTACT

532
    Please subscribe to the Hive mailing list:  http://listserver.ebi.ac.uk/mailman/listinfo/ehive-users  to discuss Hive-related questions or to be notified of our updates
Leo Gordon's avatar
Leo Gordon committed
533

534 535
=cut