Graph.pm 20.3 KB
Newer Older
1
=pod
2
3
4

=head1 NAME

5
    Bio::EnsEMBL::Hive::Utils::Graph
6
7
8

=head1 SYNOPSIS

9
10
11
12
    my $dba = get_hive_dba();
    my $g = Bio::EnsEMBL::Hive::Utils::Graph->new(-DBA => $dba);
    my $graphviz = $g->build();
    $graphviz->as_png('location.png');
13
14
15

=head1 DESCRIPTION

16
17
18
19
20
21
22
    This is a module for converting a hive database's flow of analyses, control 
    rules and dataflows into the GraphViz model language. This information can
    then be converted to an image or to the dot language for further manipulation
    in GraphViz.

=head1 LICENSE

23
    Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
24
25
26
27
28
29
30
31
32

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software distributed under the License
    is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and limitations under the License.
33

34
=head1 CONTACT
35

36
  Please subscribe to the Hive mailing list:  http://listserver.ebi.ac.uk/mailman/listinfo/ehive-users  to discuss Hive-related questions or to be notified of our updates
37
38
39
40
41

=head1 APPENDIX

    The rest of the documentation details each of the object methods.
    Internal methods are usually preceded with a _
42
43
44

=cut

45
46
47

package Bio::EnsEMBL::Hive::Utils::Graph;

48
49
50
use strict;
use warnings;

51
use Bio::EnsEMBL::Hive::Utils::GraphViz;
52
use Bio::EnsEMBL::Hive::Utils::Collection;
53
use Bio::EnsEMBL::Hive::Utils::Config;
54

55
56
use base ('Bio::EnsEMBL::Hive::Configurable');

57
58
59

=head2 new()

60
61
62
63
64
65
  Arg [1] : Bio::EnsEMBL::Hive::DBSQL::DBAdaptor $dba;
              The adaptor to get information from
  Arg [2] : (optional) string $config_file_name;
                  A JSON file name to initialize the Config object with.
                  If one is not given then we don't pass anything into Config's constructor,
                  which results in loading configuration from Config's standard locations.
66
67
68
69
70
71
72
  Returntype : Graph object
  Exceptions : If the parameters are not as required
  Status     : Beta
  
=cut

sub new {
73
  my ($class, $dba, $config_file_name) = @_;
74

75
  my $self = bless({}, ref($class) || $class);
76

77
  $self->dba($dba);
78
  my $config = Bio::EnsEMBL::Hive::Utils::Config->new( $config_file_name ? $config_file_name : () );
79
  $self->config($config);
80
  $self->context( [ 'Graph' ] );
81

82
83
84
85
86
87
88
89
90
91
92
93
94
95
  return $self;
}


=head2 graph()

  Arg [1] : The GraphViz instance created by this module
  Returntype : GraphViz
  Exceptions : None
  Status     : Beta

=cut

sub graph {
96
97
98
99
100
101
102
    my ($self) = @_;

    if(! exists $self->{graph}) {
        my $padding  = $self->config_get('Pad') || 0;
        $self->{graph} = Bio::EnsEMBL::Hive::Utils::GraphViz->new( name => 'AnalysisWorkflow', ratio => qq{compress"; pad = "$padding}  ); # injection hack!
    }
    return $self->{graph};
103
104
105
106
107
108
109
110
111
112
113
114
115
}


=head2 dba()

  Arg [1] : The DBAdaptor instance
  Returntype : DBAdaptor
  Exceptions : If the given object is not a hive DBAdaptor
  Status     : Beta

=cut

sub dba {
116
117
118
119
120
121
122
    my $self = shift @_;

    if(@_) {
        $self->{dba} = shift @_;
    }

    return $self->{dba};
123
124
125
}


126
sub _analysis_node_name {
127
    my $analysis = shift @_;
128

129
130
#    return 'analysis_' . $analysis->dbID;
    return 'analysis_' . $analysis->logic_name;
131
132
}

133
sub _table_node_name {
134
    my ($self, $df_rule) = @_;
135

136
137
    return 'table_' . $df_rule->to_analysis->table_name .
                ($self->config_get('DuplicateTables') ?  '_'.$df_rule->from_analysis->logic_name : '');
138
139
}

140

141
142
143
144
145
146
sub _midpoint_name {
    my $rule_id = shift @_;

    return 'dfr_'.$rule_id.'_mp';
}

147
148
149
150
151
152
153
154
155
156
157

=head2 build()

  Returntype : The GraphViz object built & populated
  Exceptions : Raised if there are issues with accessing the database
  Description : Builds the graph object and returns it.
  Status     : Beta

=cut

sub build {
158
    my ($self) = @_;
159

160
161
162
    my $all_analyses_coll       = Bio::EnsEMBL::Hive::Utils::Collection->new( $self->dba()->get_AnalysisAdaptor()->fetch_all );
    my $all_control_rules_coll  = Bio::EnsEMBL::Hive::Utils::Collection->new( $self->dba()->get_AnalysisCtrlRuleAdaptor()->fetch_all );
    my $all_dataflow_rules_coll = Bio::EnsEMBL::Hive::Utils::Collection->new( $self->dba()->get_DataflowRuleAdaptor()->fetch_all );
163

164
165
166
167
168
    foreach my $c_rule ( $all_control_rules_coll->list ) {
        my $ctrled_analysis = $all_analyses_coll->find_one_by('dbID', $c_rule->ctrled_analysis_id );
        $c_rule->ctrled_analysis( $ctrled_analysis );
        push @{$ctrled_analysis->control_rules_collection}, $c_rule;
    }
169

170
171
172
173
174
175
176
177
178
179
180
    foreach my $df_rule ( $all_dataflow_rules_coll->list ) {
        my $from_analysis = $all_analyses_coll->find_one_by('dbID', $df_rule->from_analysis_id );
        $df_rule->from_analysis( $from_analysis );
        push @{$from_analysis->dataflow_rules_collection}, $df_rule;

        if(my $target_object = $all_analyses_coll->find_one_by('logic_name', $df_rule->to_analysis_url )) {
            $df_rule->to_analysis( $target_object );
            if(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::Analysis')) {
                $target_object->{'_inflow_count'}++;
            }
        } # otherwise it may be a link out (unsupported at the moment)
181
182
183
184
185

        if( my $funnel_dataflow_rule_id  = $df_rule->funnel_dataflow_rule_id ) {
            my $funnel_dataflow_rule = $all_dataflow_rules_coll->find_one_by('dbID', $funnel_dataflow_rule_id );
            $df_rule->funnel_dataflow_rule( $funnel_dataflow_rule );
        }
186
187
    }

188
    my %subgraph_allocation = ();   # maps node names to midpoint names of the funnel dataflow rule (or null if toplevel)
189
190
191

        # NB: this is a very approximate algorithm with rough edges!
        # It will not find all start nodes in cyclic components!
192
193
    foreach my $source_analysis ( $all_analyses_coll->list ) {
        unless( $source_analysis->{'_inflow_count'} ) {    # if there is no dataflow into this analysis
194
                # run the recursion in each component that has a non-cyclic start:
195
            $self->_allocate_to_subgraph( $source_analysis, \%subgraph_allocation );
196
197
198
199
        }
    }

    $self->_add_hive_details();
200
201
202
203
204
205
    foreach my $analysis ( $all_analyses_coll->list ) {
        $self->_add_analysis_node($analysis);
    }
    foreach my $analysis ( $all_analyses_coll->list ) {
        $self->_control_rules( $analysis->control_rules_collection );
        $self->_dataflow_rules( $analysis->dataflow_rules_collection, \%subgraph_allocation );
206
207
    }

208
    if($self->config_get('DisplayStretched') ) {
209
        while( my($from, $to) = each %subgraph_allocation) {
210
211
            if($to && $from=~/^analysis/) {
                $self->graph->add_edge( $from => $to,
212
213
214
215
216
217
218
                    color     => 'black',
                    style     => 'invis',   # toggle visibility by changing 'invis' to 'dashed'
                );
            }
        }
    }

219
    if($self->config_get('DisplaySemaphoreBoxes') ) {
220
        $self->graph->subgraphs( \%subgraph_allocation );
221
222
        $self->graph->colour_scheme( $self->config_get('Box', 'ColourScheme') );
        $self->graph->colour_offset( $self->config_get('Box', 'ColourOffset') );
223
224
225
226
227
228
229
    }

    return $self->graph();
}


sub _allocate_to_subgraph {
230
    my ($self, $source_analysis, $subgraph_allocation ) = @_;
231

232
    my $source_analysis_allocation  = $subgraph_allocation->{ _analysis_node_name( $source_analysis ) };  # for some analyses it will be undef
233

234
    foreach my $df_rule ( @{ $source_analysis->dataflow_rules_collection } ) {    # this will only work if the analyses objects are ALL cached before loading DFRs
235
        my $target_object       = $df_rule->to_analysis();
236
        my $target_node_name;
237

238
        if(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::Analysis')) {
239
            $target_node_name = _analysis_node_name( $target_object );
240
        } elsif(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::NakedTable')) {
241
            $target_node_name = $self->_table_node_name( $df_rule );
242
        } elsif(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::Accumulator')) {
243
            next;
244
245
246
        } else {
            warn('Do not know how to handle the type '.ref($target_object));
            next;
247
        }
248

249
        my $proposed_allocation;    # will depend on whether we start a new semaphore
250
251
        my $funnel_dataflow_rule  = $df_rule->funnel_dataflow_rule();
        if( $funnel_dataflow_rule ) {
252
            $proposed_allocation =
253
                _midpoint_name( $funnel_dataflow_rule->dbID );       # if we do start a new semaphore, report to the new funnel (based on common funnel rule's midpoint)
254

255
            my $fan_midpoint_name = _midpoint_name( $df_rule->dbID );
256
257
            $subgraph_allocation->{ $fan_midpoint_name } = $proposed_allocation;

258
            my $funnel_midpoint_name = _midpoint_name( $funnel_dataflow_rule->dbID );
259
260
261
            $subgraph_allocation->{ $funnel_midpoint_name } = $source_analysis_allocation;   # draw the funnel's midpoint outside of the box
        } else {
            $proposed_allocation = $source_analysis_allocation;   # if we don't start a new semaphore, inherit the allocation of the source
262
        }
263

264
265
266
            # we allocate on first-come basis at the moment:
        if( exists $subgraph_allocation->{ $target_node_name } ) {  # already allocated?
            my $known_allocation = $subgraph_allocation->{ $target_node_name } || '';
267
268
269
            $proposed_allocation ||= '';

            if( $known_allocation eq $proposed_allocation) {
270
                # warn "analysis '$target_node_name' has already been allocated to the same '$known_allocation' by another branch";
271
            } else {
272
273
274
                # warn "analysis '$target_node_name' has already been allocated to '$known_allocation' however this branch would allocate it to '$proposed_allocation'";
            }

275
            if($funnel_dataflow_rule) {  # correction for multiple entries into the same box (probably needs re-thinking)
276
                my $fan_midpoint_name = _midpoint_name( $df_rule->dbID );
277
                $subgraph_allocation->{ $fan_midpoint_name } = $subgraph_allocation->{ $target_node_name };
278
279
280
            }

        } else {
281
282
            # warn "allocating analysis '$target_node_name' to '$proposed_allocation'";
            $subgraph_allocation->{ $target_node_name } = $proposed_allocation;
283

284
285
286
            if(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::Analysis')) {
                $self->_allocate_to_subgraph( $target_object, $subgraph_allocation );
            }
287
288
        }
    }
289
290
}

291

292
293
sub _add_hive_details {
  my ($self) = @_;
294

295
  my $node_fontname  = $self->config_get('Node', 'Details', 'Font');
296

297
  if( $self->config_get('DisplayDetails') ) {
298
    my $dbc = $self->dba()->dbc();
299
    my $label = sprintf('%s@%s', $dbc->dbname, $dbc->host || '-');
300
    $self->graph()->add_node( 'Details',
301
302
303
      label     => $label,
      fontname  => $node_fontname,
      shape     => 'plaintext',
304
305
306
307
    );
  }
}

308

309
sub _add_analysis_node {
310
    my ($self, $analysis) = @_;
311

312
    my $analysis_stats = $analysis->stats();
313

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
    my ($breakout_label, $total_job_count, $count_hash)   = $analysis_stats->job_count_breakout();
    my $analysis_status                                   = $analysis_stats->status;
    my $analysis_status_colour                            = $self->config_get('Node', 'AnalysisStatus', $analysis_status, 'Colour');
    my $style                                             = $analysis->can_be_empty() ? 'dashed, filled' : 'filled' ;
    my $node_fontname                                     = $self->config_get('Node', 'AnalysisStatus', $analysis_status, 'Font');
    my $display_stats                                     = $self->config_get('DisplayStats');

    my $colspan = 0;
    my $bar_chart = '';

    if( $display_stats eq 'barchart' ) {
        foreach my $count_method (qw(SEMAPHORED READY INPROGRESS DONE FAILED)) {
            if(my $count=$count_hash->{lc($count_method).'_job_count'}) {
                $bar_chart .= '<td bgcolor="'.$self->config_get('Node', 'JobStatus', $count_method, 'Colour').'" width="'.int(100*$count/$total_job_count).'%">'.$count.lc(substr($count_method,0,1)).'</td>';
                ++$colspan;
            }
        }
        if($colspan != 1) {
            $bar_chart .= '<td>='.$total_job_count.'</td>';
            ++$colspan;
        }
    }

    $colspan ||= 1;
338
    my $analysis_label  = '<<table border="0" cellborder="0" cellspacing="0" cellpadding="1"><tr><td colspan="'.$colspan.'">'.$analysis->logic_name().' ('.$analysis->dbID.')</td></tr>';
339
340
341
342
343
344
345
346
347
    if( $display_stats ) {
        $analysis_label    .= qq{<tr><td colspan="$colspan"> </td></tr>};
        if( $display_stats eq 'barchart') {
            $analysis_label    .= qq{<tr>$bar_chart</tr>};
        } elsif( $display_stats eq 'text') {
            $analysis_label    .= qq{<tr><td colspan="$colspan">$breakout_label</td></tr>};
        }
    }

348
    if( my $job_limit = $self->config_get('DisplayJobs') ) {
349
        my $adaptor = $self->dba->get_AnalysisJobAdaptor();
350
351
352
353
354
355
356
        my @jobs = sort {$a->dbID <=> $b->dbID} @{ $adaptor->fetch_some_by_analysis_id_limit( $analysis->dbID, $job_limit+1 )};

        my $hit_limit;
        if(scalar(@jobs)>$job_limit) {
            pop @jobs;
            $hit_limit = 1;
        }
357
358
359
360
361
362
363
364
365
366
367

        $analysis_label    .= '<tr><td colspan="'.$colspan.'"> </td></tr>';
        foreach my $job (@jobs) {
            my $input_id = $job->input_id;
            my $status   = $job->status;
            my $job_id   = $job->dbID;
            $input_id=~s/\>/&gt;/g;
            $input_id=~s/\</&lt;/g;
            $input_id=~s/\{|\}//g;
            $analysis_label    .= qq{<tr><td colspan="$colspan" bgcolor="}.$self->config_get('Node', 'JobStatus', $status, 'Colour').qq{">$job_id [$status]: $input_id</td></tr>};
        }
368
369
370
371

        if($hit_limit) {
            $analysis_label    .= qq{<tr><td colspan="$colspan">[ and }.($total_job_count-$job_limit).qq{ more ]</td></tr>};
        }
372
373
    }
    $analysis_label    .= '</table>>';
374
  
375
    $self->graph->add_node( _analysis_node_name( $analysis ),
376
377
378
379
380
381
        label       => $analysis_label,
        shape       => 'record',
        fontname    => $node_fontname,
        style       => $style,
        fillcolor   => $analysis_status_colour,
    );
382
383
384
385
}


sub _control_rules {
386
  my ($self, $ctrl_rules) = @_;
387
  
388
  my $control_colour = $self->config_get('Edge', 'Control', 'Colour');
389
390
  my $graph = $self->graph();

391
      #The control rules are always from and to an analysis so no need to search for odd cases here
392
393
394
  foreach my $c_rule ( @$ctrl_rules ) {
    my $from_node_name = _analysis_node_name( $c_rule->condition_analysis );
    my $to_node_name   = _analysis_node_name( $c_rule->ctrled_analysis );
395
396

    $graph->add_edge( $from_node_name => $to_node_name,
397
      color => $control_colour,
398
      arrowhead => 'tee',
399
400
    );
  }
401
402
}

403

404
sub _dataflow_rules {
405
    my ($self, $dataflow_rules, $subgraph_allocation) = @_;
406

407
    my $graph = $self->graph();
408
409
410
411
    my $dataflow_colour     = $self->config_get('Edge', 'Data', 'Colour');
    my $semablock_colour    = $self->config_get('Edge', 'Semablock', 'Colour');
    my $accu_colour         = $self->config_get('Edge', 'Accu', 'Colour');
    my $df_edge_fontname    = $self->config_get('Edge', 'Data', 'Font');
412
413

    my %needs_a_midpoint = ();
414
    foreach my $df_rule ( @$dataflow_rules ) {
415
        if( my $funnel_dataflow_rule = $df_rule->funnel_dataflow_rule ) {
416
            $needs_a_midpoint{ $df_rule->dbID }++;
417
            $needs_a_midpoint{ $funnel_dataflow_rule->dbID }++;
418
419
        }
    }
420

421
    foreach my $df_rule ( @$dataflow_rules ) {
422
    
423
424
        my ($rule_id, $from_analysis, $branch_code, $funnel_dataflow_rule, $target_object) =
            ($df_rule->dbID, $df_rule->from_analysis, $df_rule->branch_code, $df_rule->funnel_dataflow_rule, $df_rule->to_analysis);
425
        my $from_node_name = _analysis_node_name( $from_analysis );
426
        my $target_node_name;
427
    
428
            # Different treatment for analyses and tables:
429
        if(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::Analysis')) {
430
            $target_node_name = _analysis_node_name( $target_object );
431
        } elsif(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::NakedTable')) {
432

433
            $target_node_name = $self->_table_node_name( $df_rule );
434

435
            $self->_add_table_node($target_node_name, $target_object->table_name);
436
        } elsif(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::Accumulator')) {
437
            $target_node_name = $subgraph_allocation->{$from_node_name};
438

439
        } else {
440
            warn('Do not know how to handle the type '.ref($target_object));
441
442
443
444
            next;
        }

        if($needs_a_midpoint{$rule_id}) {
445
            my $midpoint_name = _midpoint_name( $rule_id );
446

447
            $graph->add_node( $midpoint_name,   # midpoint itself
448
                color       => $dataflow_colour,
449
450
                label       => '',
                shape       => 'point',
451
452
453
                fixedsize   => 1,
                width       => 0.01,
                height      => 0.01,
454
            );
455
            $graph->add_edge( $from_node_name => $midpoint_name, # first half of the two-part arrow
456
                color       => $dataflow_colour,
457
                arrowhead   => 'none',
458
                fontname    => $df_edge_fontname,
459
460
                fontcolor   => $dataflow_colour,
                label       => '#'.$branch_code,
461
            );
462
            $graph->add_edge( $midpoint_name => $target_node_name,   # second half of the two-part arrow
463
                color     => $dataflow_colour,
464
            );
465
466
            if($funnel_dataflow_rule) {
                $graph->add_edge( $midpoint_name => _midpoint_name( $funnel_dataflow_rule->dbID ),   # semaphore inter-rule link
467
                    color     => $semablock_colour,
468
469
470
471
472
473
                    style     => 'dashed',
                    arrowhead => 'tee',
                    dir       => 'both',
                    arrowtail => 'crow',
                );
            }
474
        } elsif(UNIVERSAL::isa($target_object, 'Bio::EnsEMBL::Hive::Accumulator')) {
475
                # one-part dashed arrow:
476
            $graph->add_edge( $from_node_name => $target_node_name,
477
478
                color       => $accu_colour,
                style       => 'dashed',
479
                label       => $target_object->struct_name().'#'.$branch_code,
480
481
482
483
484
                fontname    => $df_edge_fontname,
                fontcolor   => $accu_colour,
                dir         => 'both',
                arrowtail   => 'crow',
            );
485
        } else {
486
                # one-part solid arrow:
487
            $graph->add_edge( $from_node_name => $target_node_name,
488
                color       => $dataflow_colour,
489
                fontname    => $df_edge_fontname,
490
491
                fontcolor   => $dataflow_colour,
                label       => '#'.$branch_code,
492
            );
493
        } # /if($needs_a_midpoint{$rule_id})
494
    } # /foreach my $df_rule (@$dataflow_rules)
495

496
497
}

498

499
sub _add_table_node {
500
    my ($self, $table_node_name, $table_name) = @_;
501

502
    my $node_fontname    = $self->config_get('Node', 'Table', 'Font');
503
    my (@column_names, $columns, $table_data, $data_limit, $hit_limit);
504

505
    if( $data_limit = $self->config_get('DisplayData') ) {
506
507
        my $adaptor = $self->dba->get_NakedTableAdaptor();
        $adaptor->table_name( $table_name );
508

509
510
        @column_names = sort keys %{$adaptor->column_set};
        $columns = scalar(@column_names);
511
512
513
514
515
516
        $table_data = $adaptor->fetch_all( 'LIMIT '.($data_limit+1) );

        if(scalar(@$table_data)>$data_limit) {
            pop @$table_data;
            $hit_limit = 1;
        }
517
518
519
520
521
522
523
524
525
526
    }

    my $table_label = '<<table border="0" cellborder="0" cellspacing="0" cellpadding="1"><tr><td colspan="'.($columns||1).'">'.$table_name.'</td></tr>';

    if( $self->config_get('DisplayData') ) {
        $table_label .= '<tr><td colspan="'.$columns.'"> </td></tr>';
        $table_label .= '<tr>'.join('', map { qq{<td bgcolor="lightblue" border="1">$_</td>} } @column_names).'</tr>';
        foreach my $row (@$table_data) {
            $table_label .= '<tr>'.join('', map { qq{<td>$_</td>} } @{$row}{@column_names}).'</tr>';
        }
527
528
529
        if($hit_limit) {
            $table_label  .= qq{<tr><td colspan="$columns">[ more data ]</td></tr>};
        }
530
531
532
    }
    $table_label .= '</table>>';

533
    $self->graph()->add_node( $table_node_name, 
534
535
536
537
538
        label => $table_label,
        shape => 'record',
        fontname => $node_fontname,
        color => $self->config_get('Node', 'Table', 'Colour'),
    );
539
540
}

Leo Gordon's avatar
Leo Gordon committed
541
1;