init_pipeline.pl 5.93 KB
Newer Older
Leo Gordon's avatar
Leo Gordon committed
1
#!/usr/bin/env perl
2
3
4
5

# A generic loader of hive pipelines

use strict;
Leo Gordon's avatar
Leo Gordon committed
6
use warnings;
7
8
use DBI;
use Getopt::Long;
Leo Gordon's avatar
Leo Gordon committed
9
10
use Data::Dumper;                   # NB: in this case it is not for testing but for actual data structure stringification
use Bio::EnsEMBL::Utils::Argument;  # import 'rearrange()'
11
use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
12
use Bio::EnsEMBL::Hive::Extensions;
13
14
15

sub main {

16
17
    my $topup_flag  = 0;  # do not run initial scripts and only add new analyses+jobs (ignore the fetchable analyses)
    my $config_file = '';
18
19

    GetOptions(
20
               'topup=i'    => \$topup_flag,
21
22
23
24
25
               'conf=s'     => \$config_file,
    );

    unless($config_file and (-f $config_file)) {
        warn "Please supply a valid pipeline configuration file using '-conf' option\n";
Leo Gordon's avatar
Leo Gordon committed
26
        warn "Usage example:\n\t$0 -conf ../docs/long_mult_pipeline.conf\n";
27
28
29
30
31
        exit(1);
    }

    my $self = bless { do $config_file };

32
33
34
35
36
37
38
39
    unless($topup_flag) {
        foreach my $cmd (@{$self->{-pipeline_create_commands}}) {
            warn "Running the command:\n\t$cmd\n";
            if(my $retval = system($cmd)) {
                die "Return value = $retval, possibly an error\n";
            } else {
                warn "Done.\n\n";
            }
40
41
42
        }
    }

Leo Gordon's avatar
Leo Gordon committed
43
44
45
    my $hive_dba                     = new Bio::EnsEMBL::Hive::DBSQL::DBAdaptor(%{$self->{-pipeline_db}});
    my $resource_description_adaptor = $hive_dba->get_ResourceDescriptionAdaptor;
    
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

        # pre-load the resource_description table
    if($self->{-resource_classes}) {
        warn "Loading the ResourceDescriptions ...\n";

        while( my($rc_id, $mt2param) = each %{$self->{-resource_classes}} ) {
            my $description = delete $mt2param->{-desc};
            while( my($meadow_type, $xparams) = each %$mt2param ) {
                $resource_description_adaptor->create_new(
                    -RC_ID       => $rc_id,
                    -MEADOW_TYPE => $meadow_type,
                    -PARAMETERS  => $xparams,
                    -DESCRIPTION => $description,
                );
            }
Leo Gordon's avatar
Leo Gordon committed
61
        }
62
63

        warn "Done.\n\n";
Leo Gordon's avatar
Leo Gordon committed
64
65
66
    }

    my $analysis_adaptor             = $hive_dba->get_AnalysisAdaptor;
67
68

        # tune Data::Dumper module to produce the output we want:
69
70
    $Data::Dumper::Indent     = 0;  # we want everything on one line
    $Data::Dumper::Terse      = 1;  # and we want it without dummy variable names
71
72

    foreach my $aha (@{$self->{-pipeline_analyses}}) {
Leo Gordon's avatar
Leo Gordon committed
73
74
        my ($logic_name, $module, $parameters, $input_ids, $blocked, $batch_size, $hive_capacity, $rc_id) =
             rearrange([qw(logic_name module parameters input_ids blocked batch_size hive_capacity rc_id)], %$aha);
75
76
77
78
79

        if($topup_flag and $analysis_adaptor->fetch_by_logic_name($logic_name)) {
            warn "Skipping already existing analysis '$logic_name'\n";
            next;
        }
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

        warn "Creating '$logic_name'...\n";

        my $analysis = Bio::EnsEMBL::Analysis->new (
            -db              => '',
            -db_file         => '',
            -db_version      => '1',
            -logic_name      => $logic_name,
            -module          => $module,
            -parameters      => Dumper($parameters),
        );

        $analysis_adaptor->store($analysis);

        my $stats = $analysis->stats();
        $stats->batch_size( $batch_size )       if(defined($batch_size));

Leo Gordon's avatar
Leo Gordon committed
97
98
# ToDo: hive_capacity for some analyses is set to '-1' (i.e. "not limited")
# Do we want this behaviour BY DEFAULT?
99
100
        $stats->hive_capacity( $hive_capacity ) if(defined($hive_capacity));

Leo Gordon's avatar
Leo Gordon committed
101
102
        $stats->rc_id( $rc_id ) if(defined($rc_id));

103
104
105
106
107
108
109
            # some analyses will be waiting for human intervention in blocked state:
        $stats->status($blocked ? 'BLOCKED' : 'READY');
        $stats->update();

            # now create the corresponding jobs (if there are any):
        foreach my $input_id (@$input_ids) {

110
            Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor->CreateNewJob(
111
112
                -input_id       => Dumper($input_id),
                -analysis       => $analysis,
113
                -input_job_id   => 0, # because these jobs are created by the initialization script, not by another job
114
115
            );
        }
116
117
118
119
    }

        # Now, run separately through the already created analyses and link them together:
        #
Leo Gordon's avatar
Leo Gordon committed
120
121
    my $ctrl_rule_adaptor            = $hive_dba->get_AnalysisCtrlRuleAdaptor;
    my $dataflow_rule_adaptor        = $hive_dba->get_DataflowRuleAdaptor;
122
123

    foreach my $aha (@{$self->{-pipeline_analyses}}) {
Leo Gordon's avatar
Leo Gordon committed
124
125
        my ($logic_name, $wait_for, $flow_into) =
             rearrange([qw(logic_name wait_for flow_into)], %$aha);
126
127

        my $analysis = $analysis_adaptor->fetch_by_logic_name($logic_name);
128
129
130

            # create control rules:
        foreach my $condition_logic_name (@$wait_for) {
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
            if(my $condition_analysis = $analysis_adaptor->fetch_by_logic_name($condition_logic_name)) {
                $ctrl_rule_adaptor->create_rule( $condition_analysis, $analysis);
                warn "Created Control rule: $condition_logic_name -| $logic_name\n";
            } else {
                die "Could not fetch analysis '$condition_logic_name' to create a control rule";
            }
        }
        foreach my $heir (@$flow_into) {
            my ($heir_logic_name, $branch) = (ref($heir) eq 'ARRAY') ? (@$heir, 1) : ($heir, 1);

            if(my $heir_analysis = $analysis_adaptor->fetch_by_logic_name($heir_logic_name)) {
                $dataflow_rule_adaptor->create_rule( $analysis, $heir_analysis, $branch);
                warn "Created DataFlow rule: $logic_name -> $heir_logic_name (branch=$branch)\n";
            } else {
                die "Could not fetch analysis '$heir_logic_name' to create a dataflow rule";
            }
147
148
149
150
        }
    }

    print "\n\n\tPlease run the following commands:\n\n";
151
152
    print "  beekeeper.pl -url ".dbconn_2_url($self->{-pipeline_db})." -sync\n";
    print "  beekeeper.pl -url ".dbconn_2_url($self->{-pipeline_db})." -loop\n";
153
154
155
156
}

main();