Commit 2796654c authored by Leo Gordon's avatar Leo Gordon
Browse files

Added coloured barchart display option and jobs/data display option (no big...

Added coloured barchart display option and jobs/data display option (no big data checks, use with care on small examples).
'Pad' is now configurable from JSON.
Beware: JSON config options have moved around!
parent 426e9d67
...@@ -27,19 +27,34 @@ ...@@ -27,19 +27,34 @@
}, },
}, },
"Graph" : { # configure diagram-generation options here: "Graph" : { # configure diagram-generation options here:
"Pad" : "1.0", # empty margin around the whole diagram
"Node" : { "Node" : {
"Font" : "Times-Roman", "Font" : "Times-Roman",
"Colour" : "white", "Colour" : "cyan",
"READY" : { "Colour" : "yellow" }, # immediately claimable
"WORKING" : { "Colour" : "yellow" }, # immediately claimable
"LOADING" : { "Colour" : "orange" }, # need a sync to claim
"ALL_CLAIMED" : { "Colour" : "orange" }, # need a sync to claim
"SYNCHING" : { "Colour" : "orange" }, # need a sync to complete to claim
"BLOCKED" : { "Colour" : "grey" }, # need a sync that may or may not unblock
"DONE" : { "Colour" : "green" },
"FAILED" : { "Colour" : "red" },
"Table" : { "Colour" : "black", "Font" : "Courier" }, "Table" : { "Colour" : "black", "Font" : "Courier" },
"Details" : { "Font" : "Helvetica" }, "Details" : { "Font" : "Helvetica" },
"JobStatus" : {
"Colour" : "yellow",
"SEMAPHORED" : { "Colour" : "red" },
"READY" : { "Colour" : "orange" },
"INPROGRESS" : { "Colour" : "yellow" },
"DONE" : { "Colour" : "green" },
"FAILED" : { "Colour" : "grey" },
},
"AnalysisStatus" : {
"EMPTY" : { "Colour" : "white" }, # unpopulated
"BLOCKED" : { "Colour" : "red" }, # need a sync that may or may not unblock
"LOADING" : { "Colour" : "orange" }, # need a sync to claim
"ALL_CLAIMED" : { "Colour" : "orange" }, # need a sync to claim
"SYNCHING" : { "Colour" : "orange" }, # need a sync to complete to claim
"READY" : { "Colour" : "yellow" }, # immediately claimable
"WORKING" : { "Colour" : "yellow" }, # immediately claimable
"DONE" : { "Colour" : "green" },
"FAILED" : { "Colour" : "grey" },
},
}, },
"Edge" : { "Edge" : {
"Font" : "Helvetica", "Font" : "Helvetica",
...@@ -51,9 +66,14 @@ ...@@ -51,9 +66,14 @@
"ColourScheme" : "blues9", # Examples: "blues9" or "bugn7" or "orrd8" or "purples7" or "ylorbr9" "ColourScheme" : "blues9", # Examples: "blues9" or "bugn7" or "orrd8" or "purples7" or "ylorbr9"
"ColourOffset" : 1, "ColourOffset" : 1,
}, },
"DisplayStats" : "barchart", # show analysis statistics using "barchart", "text" or "" (don't show statistics)
"DisplayJobs" : 0,
"DisplayData" : 0,
"DisplayDetails" : 1, "DisplayDetails" : 1,
"DisplayStretched" : 0, "DisplayStretched" : 0,
"DisplaySemaphoreBoxes" : 1, "DisplaySemaphoreBoxes" : 1,
"DuplicateTables" : 0, "DuplicateTables" : 0,
} }
} }
...@@ -73,11 +73,13 @@ sub new { ...@@ -73,11 +73,13 @@ sub new {
=cut =cut
sub graph { sub graph {
my ($self) = @_; my ($self) = @_;
if(! exists $self->{graph}) {
$self->{graph} = Bio::EnsEMBL::Hive::Utils::GraphViz->new( name => 'AnalysisWorkflow', ratio => 'compress"; pad = "1.0' ); # injection hack! if(! exists $self->{graph}) {
} my $padding = $self->config_get('Pad') || 0;
return $self->{graph}; $self->{graph} = Bio::EnsEMBL::Hive::Utils::GraphViz->new( name => 'AnalysisWorkflow', ratio => qq{compress"; pad = "$padding} ); # injection hack!
}
return $self->{graph};
} }
...@@ -256,24 +258,68 @@ sub _add_hive_details { ...@@ -256,24 +258,68 @@ sub _add_hive_details {
sub _add_analysis_node { sub _add_analysis_node {
my ($self, $a) = @_; my ($self, $analysis) = @_;
my $stats = $a->stats(); my $analysis_stats = $analysis->stats();
my ($breakout_label) = $stats->job_count_breakout(); my ($breakout_label, $total_job_count, $count_hash) = $analysis_stats->job_count_breakout();
my $analysis_status = $analysis_stats->status;
my $analysis_label = $a->logic_name().' ('.$a->dbID().')\n'.$breakout_label; my $analysis_status_colour = $self->config_get('Node', 'AnalysisStatus', $analysis_status, 'Colour');
my $shape = $a->can_be_empty() ? 'doubleoctagon' : 'ellipse' ; my $style = $analysis->can_be_empty() ? 'dashed, filled' : 'filled' ;
my $status_colour = $self->config_get('Node', $stats->status, 'Colour'); my $node_fontname = $self->config_get('Node', 'AnalysisStatus', $analysis_status, 'Font');
my $node_fontname = $self->config_get('Node', $stats->status, 'Font'); my $display_stats = $self->config_get('DisplayStats');
my $colspan = 0;
my $bar_chart = '';
if( $display_stats eq 'barchart' ) {
foreach my $count_method (qw(SEMAPHORED READY INPROGRESS DONE FAILED)) {
if(my $count=$count_hash->{lc($count_method).'_job_count'}) {
$bar_chart .= '<td bgcolor="'.$self->config_get('Node', 'JobStatus', $count_method, 'Colour').'" width="'.int(100*$count/$total_job_count).'%">'.$count.lc(substr($count_method,0,1)).'</td>';
++$colspan;
}
}
if($colspan != 1) {
$bar_chart .= '<td>='.$total_job_count.'</td>';
++$colspan;
}
}
$colspan ||= 1;
my $analysis_label = '<<table border="0" cellborder="0" cellspacing="0" cellpadding="1"><tr><td colspan="'.$colspan.'">'.$analysis->logic_name().' ('.$analysis->dbID().')</td></tr>';
if( $display_stats ) {
$analysis_label .= qq{<tr><td colspan="$colspan"> </td></tr>};
if( $display_stats eq 'barchart') {
$analysis_label .= qq{<tr>$bar_chart</tr>};
} elsif( $display_stats eq 'text') {
$analysis_label .= qq{<tr><td colspan="$colspan">$breakout_label</td></tr>};
}
}
if( $self->config_get('DisplayJobs') ) {
my $adaptor = $self->dba->get_AnalysisJobAdaptor();
my @jobs = sort {$a->dbID <=> $b->dbID} @{ $adaptor->fetch_all_by_analysis_id_status( $analysis->dbID )};
$analysis_label .= '<tr><td colspan="'.$colspan.'"> </td></tr>';
foreach my $job (@jobs) {
my $input_id = $job->input_id;
my $status = $job->status;
my $job_id = $job->dbID;
$input_id=~s/\>/&gt;/g;
$input_id=~s/\</&lt;/g;
$input_id=~s/\{|\}//g;
$analysis_label .= qq{<tr><td colspan="$colspan" bgcolor="}.$self->config_get('Node', 'JobStatus', $status, 'Colour').qq{">$job_id [$status]: $input_id</td></tr>};
}
}
$analysis_label .= '</table>>';
$self->graph->add_node( _analysis_node_name( $a->dbID() ), $self->graph->add_node( _analysis_node_name( $analysis->dbID() ),
label => $analysis_label, label => $analysis_label,
shape => $shape, shape => 'record',
style => 'filled', fontname => $node_fontname,
fontname => $node_fontname, style => $style,
fillcolor => $status_colour, fillcolor => $analysis_status_colour,
); );
} }
...@@ -327,9 +373,9 @@ sub _dataflow_rules { ...@@ -327,9 +373,9 @@ sub _dataflow_rules {
$to_id = $to->dbID(); $to_id = $to->dbID();
$to_node = _analysis_node_name($to_id); $to_node = _analysis_node_name($to_id);
} elsif(check_ref($to, 'Bio::EnsEMBL::Hive::NakedTable')) { } elsif(check_ref($to, 'Bio::EnsEMBL::Hive::NakedTable')) {
$to_node = $to->table_name(); $to_node = 'table_' . $to->table_name;
$to_node .= '_'.$from_analysis_id if $self->config_get('DuplicateTables'); $to_node .= '_'.$from_analysis_id if $self->config_get('DuplicateTables');
$self->_add_table_node($to_node); $self->_add_table_node($to_node, $to->table_name);
} else { } else {
warn('Do not know how to handle the type '.ref($to)); warn('Do not know how to handle the type '.ref($to));
next; next;
...@@ -378,22 +424,37 @@ sub _dataflow_rules { ...@@ -378,22 +424,37 @@ sub _dataflow_rules {
sub _add_table_node { sub _add_table_node {
my ($self, $table) = @_; my ($self, $table_node, $table_name) = @_;
my $node_fontname = $self->config_get('Node', 'Table', 'Font'); my $node_fontname = $self->config_get('Node', 'Table', 'Font');
my (@column_names, $columns, $table_data);
my $table_name = $table; if( $self->config_get('DisplayData') ) {
if( $self->config_get('DuplicateTables') ) { my $adaptor = $self->dba->get_NakedTableAdaptor();
$table =~ /^(.*)_([^_]*)$/; $adaptor->table_name( $table_name );
$table_name = $1;
}
$self->graph()->add_node( $table, @column_names = sort keys %{$adaptor->column_set};
label => $table_name.'\n', $columns = scalar(@column_names);
shape => 'tab', $table_data = $adaptor->fetch_all( );
fontname => $node_fontname, }
color => $self->config_get('Node', 'Table', 'Colour'),
); my $table_label = '<<table border="0" cellborder="0" cellspacing="0" cellpadding="1"><tr><td colspan="'.($columns||1).'">'.$table_name.'</td></tr>';
if( $self->config_get('DisplayData') ) {
$table_label .= '<tr><td colspan="'.$columns.'"> </td></tr>';
$table_label .= '<tr>'.join('', map { qq{<td bgcolor="lightblue" border="1">$_</td>} } @column_names).'</tr>';
foreach my $row (@$table_data) {
$table_label .= '<tr>'.join('', map { qq{<td>$_</td>} } @{$row}{@column_names}).'</tr>';
}
}
$table_label .= '</table>>';
$self->graph()->add_node( $table_node,
label => $table_label,
shape => 'record',
fontname => $node_fontname,
color => $self->config_get('Node', 'Table', 'Colour'),
);
} }
1; 1;
...@@ -108,7 +108,13 @@ sub _as_debug { ...@@ -108,7 +108,13 @@ sub _as_debug {
} }
$text .= "}\n"; $text .= "}\n";
# print $text; # GraphViz.pm thinks 'record' is the only shape that allows HTML-like labels,
# but newer versions of dot allow more freedom, so we patch dot input after generation:
#
$text=~s/^(\s+table_.*)"record"/$1"tab"/mg;
$text=~s/^(\s+analysis_.*)"record"/$1"Mrecord"/mg;
print $text;
return $text; return $text;
} }
......
...@@ -17,101 +17,92 @@ my $self = bless({}, __PACKAGE__); ...@@ -17,101 +17,92 @@ my $self = bless({}, __PACKAGE__);
$self->main(); $self->main();
sub main { sub main {
my ($self) = @_; my ($self) = @_;
$self->_options();
$self->_process_options(); $self->_options();
$self->_write_graph(); $self->_process_options();
$self->_write_graph();
} }
sub _options { sub _options {
my ($self) = @_; my ($self) = @_;
GetOptions( GetOptions(
'reg_conf|regfile=s' => \$self->{'reg_conf'}, # connection parameters
'reg_alias|regname=s' => \$self->{'reg_alias'}, 'reg_conf|regfile=s' => \$self->{'reg_conf'},
'url=s' => \$self->{url}, 'reg_alias|regname=s' => \$self->{'reg_alias'},
'host|dbhost=s' => \$self->{db_conf}->{'-host'}, 'url=s' => \$self->{url},
'port|dbport=i' => \$self->{db_conf}->{'-port'},
'user|dbuser=s' => \$self->{db_conf}->{'-user'}, 'f|format=s' => \$self->{format},
'password|dbpass=s' => \$self->{db_conf}->{'-pass'}, 'o|output=s' => \$self->{output},
'database|dbname=s' => \$self->{db_conf}->{'-dbname'},
'h|help' => \$self->{help},
'f|format=s' => \$self->{format}, 'm|man' => \$self->{man},
'o|output=s' => \$self->{output}, );
'h|help' => \$self->{help},
'm|man' => \$self->{man},
);
} }
sub _process_options { sub _process_options {
my ($self) = @_; my ($self) = @_;
#Check for help #Check for help
if($self->{help}) { if($self->{help}) {
pod2usage({-exitvalue => 0, -verbose => 1}); pod2usage({-exitvalue => 0, -verbose => 1});
} }
if($self->{man}) { if($self->{man}) {
pod2usage({-exitvalue => 0, -verbose => 2}); pod2usage({-exitvalue => 0, -verbose => 2});
} }
#Check for DB #Check for DB
if($self->{'reg_conf'} and $self->{'reg_alias'}) { if($self->{'reg_conf'} and $self->{'reg_alias'}) {
Bio::EnsEMBL::Registry->load_all($self->{'reg_conf'}); Bio::EnsEMBL::Registry->load_all($self->{'reg_conf'});
$self->{dba} = Bio::EnsEMBL::Registry->get_DBAdaptor($self->{'reg_alias'}, 'hive'); $self->{dba} = Bio::EnsEMBL::Registry->get_DBAdaptor($self->{'reg_alias'}, 'hive');
} } elsif($self->{url}) {
elsif($self->{url}) { $self->{dba} = Bio::EnsEMBL::Hive::URLFactory->fetch($self->{url}) || die("Unable to connect to $self->{url}\n");
$self->{dba} = Bio::EnsEMBL::Hive::URLFactory->fetch($self->{url}) || die("Unable to connect to $self->{url}\n"); } else {
} pod2usage({
elsif ( $self->{db_conf}->{'-host'} -message => 'ERROR: Connection parameters (url or reg_conf+reg_alias) need to be specified',
&& $self->{db_conf}->{'-user'} -exitvalue => 1,
&& $self->{db_conf}->{'-dbname'}) { # connect to database specified -verbose => 1
$self->{dba} = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new( %{$self->{db_conf}} ); });
} }
else {
pod2usage({
-message => 'ERROR: Connection parameters (reg_conf+reg_alias, url or dbhost+dbuser+dbname) need to be specified',
-exitvalue => 1,
-verbose => 1
});
}
if(! $self->{output}) { if(! $self->{output}) {
pod2usage({ pod2usage({
-message => 'ERROR: No -output flag given', -message => 'ERROR: No -output flag given',
-exitvalue => 1, -exitvalue => 1,
-verbose => 1 -verbose => 1
}); });
} }
if(!$self->{format}) { if(!$self->{format}) {
if($self->{output}=~/\.(\w+)$/) { if($self->{output}=~/\.(\w+)$/) {
$self->{format} = $1; $self->{format} = $1;
} else { } else {
die "Format was not set and could not guess from ".$self->output().". Please use either way to select it.\n"; die "Format was not set and could not guess from ".$self->output().". Please use either way to select it.\n";
}
} }
}
} }
sub _write_graph { sub _write_graph {
my ($self) = @_; my ($self) = @_;
my $graph = Bio::EnsEMBL::Hive::Utils::Graph->new( $self->{dba} ); my $graph = Bio::EnsEMBL::Hive::Utils::Graph->new( $self->{dba} );
my $graphviz = $graph->build(); my $graphviz = $graph->build();
my $call = q{as_}.$self->{format}; my $call = q{as_}.$self->{format};
eval {$graphviz->$call($self->{output});}; eval {$graphviz->$call($self->{output});};
if($@) { if($@) {
warn $@; warn $@;
pod2usage({ pod2usage({
-message => 'Error detected. Check '.$self->{format}.' is a valid format. Use a format name as supported by graphviz', -message => 'Error detected. Check '.$self->{format}.' is a valid format. Use a format name as supported by graphviz',
-exitvalue => 1, -exitvalue => 1,
-verbose => 1 -verbose => 1
}); });
} }
} }
__END__ __END__
=pod =pod
=head1 NAME =head1 NAME
...@@ -136,51 +127,31 @@ hive_config.json configuration file. ...@@ -136,51 +127,31 @@ hive_config.json configuration file.
=item B<--format> =item B<--format>
The format of the file output. See FORMATS for more information The format of the file output. See FORMATS for more information
=item B<--output> =item B<--output>
Location of the file to write to. Location of the file to write to.
=item B<-reg_conf> =item B<-reg_conf>
path to a Registry configuration file path to a Registry configuration file
=item B<-reg_alias> =item B<-reg_alias>
species/alias name for the Hive DBAdaptor species/alias name for the Hive DBAdaptor
=item B<-url> =item B<-url>
url defining where hive database is located url defining where hive database is located
=item B<-host>
mysql database host <machine>
=item B<-port>
mysql port number
=item B<-user>
mysql connection user <name>
=item B<-password>
mysql connection password <pass>
=item B<-database>
mysql database <name>
=back =back
=head1 FORMATS =head1 FORMATS
The script supports the same output formats as GraphViz & the accompanying The script supports the same output formats as GraphViz & the accompanying
Perl module do. However here are a list of common output formats you may Perl module do. However here are a list of common output formats you may
want to specify (png is the default). want to specify (png is the default).
=over 8 =over 8
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment