init_pipeline.pl 5.96 KB
Newer Older
1
2
3
4
5
6
7
#!/usr/local/ensembl/bin/perl -w

# A generic loader of hive pipelines

use strict;
use DBI;
use Getopt::Long;
Leo Gordon's avatar
Leo Gordon committed
8
9
use Data::Dumper;                   # NB: in this case it is not for testing but for actual data structure stringification
use Bio::EnsEMBL::Utils::Argument;  # import 'rearrange()'
10
use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
11
use Bio::EnsEMBL::Hive::Extensions;
12
13
14

sub main {

15
16
    my $topup_flag  = 0;  # do not run initial scripts and only add new analyses+jobs (ignore the fetchable analyses)
    my $config_file = '';
17
18

    GetOptions(
19
               'topup=i'    => \$topup_flag,
20
21
22
23
24
               'conf=s'     => \$config_file,
    );

    unless($config_file and (-f $config_file)) {
        warn "Please supply a valid pipeline configuration file using '-conf' option\n";
Leo Gordon's avatar
Leo Gordon committed
25
        warn "Usage example:\n\ttime $0 -conf ~lg4/work/ensembl-compara/scripts/family/family_pipeline.conf\n";
26
27
28
29
30
        exit(1);
    }

    my $self = bless { do $config_file };

31
32
33
34
35
36
37
38
    unless($topup_flag) {
        foreach my $cmd (@{$self->{-pipeline_create_commands}}) {
            warn "Running the command:\n\t$cmd\n";
            if(my $retval = system($cmd)) {
                die "Return value = $retval, possibly an error\n";
            } else {
                warn "Done.\n\n";
            }
39
40
41
        }
    }

Leo Gordon's avatar
Leo Gordon committed
42
43
44
    my $hive_dba                     = new Bio::EnsEMBL::Hive::DBSQL::DBAdaptor(%{$self->{-pipeline_db}});
    my $resource_description_adaptor = $hive_dba->get_ResourceDescriptionAdaptor;
    
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

        # pre-load the resource_description table
    if($self->{-resource_classes}) {
        warn "Loading the ResourceDescriptions ...\n";

        while( my($rc_id, $mt2param) = each %{$self->{-resource_classes}} ) {
            my $description = delete $mt2param->{-desc};
            while( my($meadow_type, $xparams) = each %$mt2param ) {
                $resource_description_adaptor->create_new(
                    -RC_ID       => $rc_id,
                    -MEADOW_TYPE => $meadow_type,
                    -PARAMETERS  => $xparams,
                    -DESCRIPTION => $description,
                );
            }
Leo Gordon's avatar
Leo Gordon committed
60
        }
61
62

        warn "Done.\n\n";
Leo Gordon's avatar
Leo Gordon committed
63
64
65
    }

    my $analysis_adaptor             = $hive_dba->get_AnalysisAdaptor;
66
67

        # tune Data::Dumper module to produce the output we want:
68
69
    $Data::Dumper::Indent     = 0;  # we want everything on one line
    $Data::Dumper::Terse      = 1;  # and we want it without dummy variable names
70
71

    foreach my $aha (@{$self->{-pipeline_analyses}}) {
Leo Gordon's avatar
Leo Gordon committed
72
73
        my ($logic_name, $module, $parameters, $input_ids, $blocked, $batch_size, $hive_capacity, $rc_id) =
             rearrange([qw(logic_name module parameters input_ids blocked batch_size hive_capacity rc_id)], %$aha);
74
75
76
77
78

        if($topup_flag and $analysis_adaptor->fetch_by_logic_name($logic_name)) {
            warn "Skipping already existing analysis '$logic_name'\n";
            next;
        }
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

        warn "Creating '$logic_name'...\n";

        my $analysis = Bio::EnsEMBL::Analysis->new (
            -db              => '',
            -db_file         => '',
            -db_version      => '1',
            -logic_name      => $logic_name,
            -module          => $module,
            -parameters      => Dumper($parameters),
        );

        $analysis_adaptor->store($analysis);

        my $stats = $analysis->stats();
        $stats->batch_size( $batch_size )       if(defined($batch_size));

Leo Gordon's avatar
Leo Gordon committed
96
97
# ToDo: hive_capacity for some analyses is set to '-1' (i.e. "not limited")
# Do we want this behaviour BY DEFAULT?
98
99
        $stats->hive_capacity( $hive_capacity ) if(defined($hive_capacity));

Leo Gordon's avatar
Leo Gordon committed
100
101
        $stats->rc_id( $rc_id ) if(defined($rc_id));

102
103
104
105
106
107
108
            # some analyses will be waiting for human intervention in blocked state:
        $stats->status($blocked ? 'BLOCKED' : 'READY');
        $stats->update();

            # now create the corresponding jobs (if there are any):
        foreach my $input_id (@$input_ids) {

109
            Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor->CreateNewJob(
110
111
                -input_id       => Dumper($input_id),
                -analysis       => $analysis,
112
                -input_job_id   => 0, # because these jobs are created by the initialization script, not by another job
113
114
            );
        }
115
116
117
118
    }

        # Now, run separately through the already created analyses and link them together:
        #
Leo Gordon's avatar
Leo Gordon committed
119
120
    my $ctrl_rule_adaptor            = $hive_dba->get_AnalysisCtrlRuleAdaptor;
    my $dataflow_rule_adaptor        = $hive_dba->get_DataflowRuleAdaptor;
121
122

    foreach my $aha (@{$self->{-pipeline_analyses}}) {
Leo Gordon's avatar
Leo Gordon committed
123
124
        my ($logic_name, $wait_for, $flow_into) =
             rearrange([qw(logic_name wait_for flow_into)], %$aha);
125
126

        my $analysis = $analysis_adaptor->fetch_by_logic_name($logic_name);
127
128
129

            # create control rules:
        foreach my $condition_logic_name (@$wait_for) {
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
            if(my $condition_analysis = $analysis_adaptor->fetch_by_logic_name($condition_logic_name)) {
                $ctrl_rule_adaptor->create_rule( $condition_analysis, $analysis);
                warn "Created Control rule: $condition_logic_name -| $logic_name\n";
            } else {
                die "Could not fetch analysis '$condition_logic_name' to create a control rule";
            }
        }
        foreach my $heir (@$flow_into) {
            my ($heir_logic_name, $branch) = (ref($heir) eq 'ARRAY') ? (@$heir, 1) : ($heir, 1);

            if(my $heir_analysis = $analysis_adaptor->fetch_by_logic_name($heir_logic_name)) {
                $dataflow_rule_adaptor->create_rule( $analysis, $heir_analysis, $branch);
                warn "Created DataFlow rule: $logic_name -> $heir_logic_name (branch=$branch)\n";
            } else {
                die "Could not fetch analysis '$heir_logic_name' to create a dataflow rule";
            }
146
147
148
149
        }
    }

    print "\n\n\tPlease run the following commands:\n\n";
150
151
    print "  beekeeper.pl -url ".dbconn_2_url($self->{-pipeline_db})." -sync\n";
    print "  beekeeper.pl -url ".dbconn_2_url($self->{-pipeline_db})." -loop\n";
152
153
154
155
}

main();