Skip to content

GitLab

Commit 2796654c authored by Leo Gordon's avatar Leo Gordon
Browse files

Added coloured barchart display option and jobs/data display option (no big... 

Added coloured barchart display option and jobs/data display option (no big data checks, use with care on small examples).
'Pad' is now configurable from JSON.
Beware: JSON config options have moved around!
parent 426e9d67
Hide whitespace changes
Inline Side-by-side
Showing with 212 additions and 154 deletions
hive_config.json
View file @ 2796654c
......@@ -27,19 +27,34 @@
},
},
"Graph" : { # configure diagram-generation options here:
"Pad" : "1.0", # empty margin around the whole diagram
"Node" : {
"Font" : "Times-Roman",
"Colour" : "white",
"READY" : { "Colour" : "yellow" }, # immediately claimable
"WORKING" : { "Colour" : "yellow" }, # immediately claimable
"LOADING" : { "Colour" : "orange" }, # need a sync to claim
"ALL_CLAIMED" : { "Colour" : "orange" }, # need a sync to claim
"SYNCHING" : { "Colour" : "orange" }, # need a sync to complete to claim
"BLOCKED" : { "Colour" : "grey" }, # need a sync that may or may not unblock
"DONE" : { "Colour" : "green" },
"FAILED" : { "Colour" : "red" },
"Colour" : "cyan",
"Table" : { "Colour" : "black", "Font" : "Courier" },
"Details" : { "Font" : "Helvetica" },
"JobStatus" : {
"Colour" : "yellow",
"SEMAPHORED" : { "Colour" : "red" },
"READY" : { "Colour" : "orange" },
"INPROGRESS" : { "Colour" : "yellow" },
"DONE" : { "Colour" : "green" },
"FAILED" : { "Colour" : "grey" },
},
"AnalysisStatus" : {
"EMPTY" : { "Colour" : "white" }, # unpopulated
"BLOCKED" : { "Colour" : "red" }, # need a sync that may or may not unblock
"LOADING" : { "Colour" : "orange" }, # need a sync to claim
"ALL_CLAIMED" : { "Colour" : "orange" }, # need a sync to claim
"SYNCHING" : { "Colour" : "orange" }, # need a sync to complete to claim
"READY" : { "Colour" : "yellow" }, # immediately claimable
"WORKING" : { "Colour" : "yellow" }, # immediately claimable
"DONE" : { "Colour" : "green" },
"FAILED" : { "Colour" : "grey" },
},
},
"Edge" : {
"Font" : "Helvetica",
......@@ -51,9 +66,14 @@
"ColourScheme" : "blues9", # Examples: "blues9" or "bugn7" or "orrd8" or "purples7" or "ylorbr9"
"ColourOffset" : 1,
},
"DisplayStats" : "barchart", # show analysis statistics using "barchart", "text" or "" (don't show statistics)
"DisplayJobs" : 0,
"DisplayData" : 0,
"DisplayDetails" : 1,
"DisplayStretched" : 0,
"DisplaySemaphoreBoxes" : 1,
"DuplicateTables" : 0,
"DuplicateTables" : 0,
}
}
modules/Bio/EnsEMBL/Hive/Utils/Graph.pm
View file @ 2796654c
......@@ -73,11 +73,13 @@ sub new {
=cut
sub graph {
my ($self) = @_;
if(! exists $self->{graph}) {
$self->{graph} = Bio::EnsEMBL::Hive::Utils::GraphViz->new( name => 'AnalysisWorkflow', ratio => 'compress"; pad = "1.0' ); # injection hack!
}
return $self->{graph};
my ($self) = @_;
if(! exists $self->{graph}) {
my $padding = $self->config_get('Pad') || 0;
$self->{graph} = Bio::EnsEMBL::Hive::Utils::GraphViz->new( name => 'AnalysisWorkflow', ratio => qq{compress"; pad = "$padding} ); # injection hack!
}
return $self->{graph};
}
......@@ -256,24 +258,68 @@ sub _add_hive_details {
sub _add_analysis_node {
my ($self, $a) = @_;
my ($self, $analysis) = @_;
my $stats = $a->stats();
my $analysis_stats = $analysis->stats();
my ($breakout_label) = $stats->job_count_breakout();
my $analysis_label = $a->logic_name().' ('.$a->dbID().')\n'.$breakout_label;
my $shape = $a->can_be_empty() ? 'doubleoctagon' : 'ellipse' ;
my $status_colour = $self->config_get('Node', $stats->status, 'Colour');
my $node_fontname = $self->config_get('Node', $stats->status, 'Font');
my ($breakout_label, $total_job_count, $count_hash) = $analysis_stats->job_count_breakout();
my $analysis_status = $analysis_stats->status;
my $analysis_status_colour = $self->config_get('Node', 'AnalysisStatus', $analysis_status, 'Colour');
my $style = $analysis->can_be_empty() ? 'dashed, filled' : 'filled' ;
my $node_fontname = $self->config_get('Node', 'AnalysisStatus', $analysis_status, 'Font');
my $display_stats = $self->config_get('DisplayStats');
my $colspan = 0;
my $bar_chart = '';
if( $display_stats eq 'barchart' ) {
foreach my $count_method (qw(SEMAPHORED READY INPROGRESS DONE FAILED)) {
if(my $count=$count_hash->{lc($count_method).'_job_count'}) {
$bar_chart .= '<td bgcolor="'.$self->config_get('Node', 'JobStatus', $count_method, 'Colour').'" width="'.int(100*$count/$total_job_count).'%">'.$count.lc(substr($count_method,0,1)).'</td>';
++$colspan;
}
}
if($colspan != 1) {
$bar_chart .= '<td>='.$total_job_count.'</td>';
++$colspan;
}
}
$colspan ||= 1;
my $analysis_label = '<<table border="0" cellborder="0" cellspacing="0" cellpadding="1"><tr><td colspan="'.$colspan.'">'.$analysis->logic_name().' ('.$analysis->dbID().')</td></tr>';
if( $display_stats ) {
$analysis_label .= qq{<tr><td colspan="$colspan"> </td></tr>};
if( $display_stats eq 'barchart') {
$analysis_label .= qq{<tr>$bar_chart</tr>};
} elsif( $display_stats eq 'text') {
$analysis_label .= qq{<tr><td colspan="$colspan">$breakout_label</td></tr>};
}
}
if( $self->config_get('DisplayJobs') ) {
my $adaptor = $self->dba->get_AnalysisJobAdaptor();
my @jobs = sort {$a->dbID <=> $b->dbID} @{ $adaptor->fetch_all_by_analysis_id_status( $analysis->dbID )};
$analysis_label .= '<tr><td colspan="'.$colspan.'"> </td></tr>';
foreach my $job (@jobs) {
my $input_id = $job->input_id;
my $status = $job->status;
my $job_id = $job->dbID;
$input_id=~s/\>/&gt;/g;
$input_id=~s/\</&lt;/g;
$input_id=~s/\{|\}//g;
$analysis_label .= qq{<tr><td colspan="$colspan" bgcolor="}.$self->config_get('Node', 'JobStatus', $status, 'Colour').qq{">$job_id [$status]: $input_id</td></tr>};
}
}
$analysis_label .= '</table>>';
$self->graph->add_node( _analysis_node_name( $a->dbID() ),
label => $analysis_label,
shape => $shape,
style => 'filled',
fontname => $node_fontname,
fillcolor => $status_colour,
);
$self->graph->add_node( _analysis_node_name( $analysis->dbID() ),
label => $analysis_label,
shape => 'record',
fontname => $node_fontname,
style => $style,
fillcolor => $analysis_status_colour,
);
}
......@@ -327,9 +373,9 @@ sub _dataflow_rules {
$to_id = $to->dbID();
$to_node = _analysis_node_name($to_id);
} elsif(check_ref($to, 'Bio::EnsEMBL::Hive::NakedTable')) {
$to_node = $to->table_name();
$to_node = 'table_' . $to->table_name;
$to_node .= '_'.$from_analysis_id if $self->config_get('DuplicateTables');
$self->_add_table_node($to_node);
$self->_add_table_node($to_node, $to->table_name);
} else {
warn('Do not know how to handle the type '.ref($to));
next;
......@@ -378,22 +424,37 @@ sub _dataflow_rules {
sub _add_table_node {
my ($self, $table) = @_;
my ($self, $table_node, $table_name) = @_;
my $node_fontname = $self->config_get('Node', 'Table', 'Font');
my $node_fontname = $self->config_get('Node', 'Table', 'Font');
my (@column_names, $columns, $table_data);
my $table_name = $table;
if( $self->config_get('DuplicateTables') ) {
$table =~ /^(.*)_([^_]*)$/;
$table_name = $1;
}
if( $self->config_get('DisplayData') ) {
my $adaptor = $self->dba->get_NakedTableAdaptor();
$adaptor->table_name( $table_name );
$self->graph()->add_node( $table,
label => $table_name.'\n',
shape => 'tab',
fontname => $node_fontname,
color => $self->config_get('Node', 'Table', 'Colour'),
);
@column_names = sort keys %{$adaptor->column_set};
$columns = scalar(@column_names);
$table_data = $adaptor->fetch_all( );
}
my $table_label = '<<table border="0" cellborder="0" cellspacing="0" cellpadding="1"><tr><td colspan="'.($columns||1).'">'.$table_name.'</td></tr>';
if( $self->config_get('DisplayData') ) {
$table_label .= '<tr><td colspan="'.$columns.'"> </td></tr>';
$table_label .= '<tr>'.join('', map { qq{<td bgcolor="lightblue" border="1">$_</td>} } @column_names).'</tr>';
foreach my $row (@$table_data) {
$table_label .= '<tr>'.join('', map { qq{<td>$_</td>} } @{$row}{@column_names}).'</tr>';
}
}
$table_label .= '</table>>';
$self->graph()->add_node( $table_node,
label => $table_label,
shape => 'record',
fontname => $node_fontname,
color => $self->config_get('Node', 'Table', 'Colour'),
);
}
1;
modules/Bio/EnsEMBL/Hive/Utils/GraphViz.pm
View file @ 2796654c
......@@ -108,7 +108,13 @@ sub _as_debug {
}
$text .= "}\n";
# print $text;
# GraphViz.pm thinks 'record' is the only shape that allows HTML-like labels,
# but newer versions of dot allow more freedom, so we patch dot input after generation:
#
$text=~s/^(\s+table_.*)"record"/$1"tab"/mg;
$text=~s/^(\s+analysis_.*)"record"/$1"Mrecord"/mg;
print $text;
return $text;
}
......
scripts/generate_graph.pl
View file @ 2796654c
......@@ -17,101 +17,92 @@ my $self = bless({}, __PACKAGE__);
$self->main();
sub main {
my ($self) = @_;
$self->_options();
$self->_process_options();
$self->_write_graph();
my ($self) = @_;
$self->_options();
$self->_process_options();
$self->_write_graph();
}
sub _options {
my ($self) = @_;
GetOptions(
'reg_conf|regfile=s' => \$self->{'reg_conf'},
'reg_alias|regname=s' => \$self->{'reg_alias'},
'url=s' => \$self->{url},
'host|dbhost=s' => \$self->{db_conf}->{'-host'},
'port|dbport=i' => \$self->{db_conf}->{'-port'},
'user|dbuser=s' => \$self->{db_conf}->{'-user'},
'password|dbpass=s' => \$self->{db_conf}->{'-pass'},
'database|dbname=s' => \$self->{db_conf}->{'-dbname'},
'f|format=s' => \$self->{format},
'o|output=s' => \$self->{output},
'h|help' => \$self->{help},
'm|man' => \$self->{man},
);
my ($self) = @_;
GetOptions(
# connection parameters
'reg_conf|regfile=s' => \$self->{'reg_conf'},
'reg_alias|regname=s' => \$self->{'reg_alias'},
'url=s' => \$self->{url},
'f|format=s' => \$self->{format},
'o|output=s' => \$self->{output},
'h|help' => \$self->{help},
'm|man' => \$self->{man},
);
}
sub _process_options {
my ($self) = @_;
#Check for help
if($self->{help}) {
pod2usage({-exitvalue => 0, -verbose => 1});
}
if($self->{man}) {
pod2usage({-exitvalue => 0, -verbose => 2});
}
#Check for DB
if($self->{'reg_conf'} and $self->{'reg_alias'}) {
Bio::EnsEMBL::Registry->load_all($self->{'reg_conf'});
$self->{dba} = Bio::EnsEMBL::Registry->get_DBAdaptor($self->{'reg_alias'}, 'hive');
}
elsif($self->{url}) {
$self->{dba} = Bio::EnsEMBL::Hive::URLFactory->fetch($self->{url}) || die("Unable to connect to $self->{url}\n");
}
elsif ( $self->{db_conf}->{'-host'}
&& $self->{db_conf}->{'-user'}
&& $self->{db_conf}->{'-dbname'}) { # connect to database specified
$self->{dba} = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new( %{$self->{db_conf}} );
}
else {
pod2usage({
-message => 'ERROR: Connection parameters (reg_conf+reg_alias, url or dbhost+dbuser+dbname) need to be specified',
-exitvalue => 1,
-verbose => 1
});
}
my ($self) = @_;
#Check for help
if($self->{help}) {
pod2usage({-exitvalue => 0, -verbose => 1});
}
if($self->{man}) {
pod2usage({-exitvalue => 0, -verbose => 2});
}
#Check for DB
if($self->{'reg_conf'} and $self->{'reg_alias'}) {
Bio::EnsEMBL::Registry->load_all($self->{'reg_conf'});
$self->{dba} = Bio::EnsEMBL::Registry->get_DBAdaptor($self->{'reg_alias'}, 'hive');
} elsif($self->{url}) {
$self->{dba} = Bio::EnsEMBL::Hive::URLFactory->fetch($self->{url}) || die("Unable to connect to $self->{url}\n");
} else {
pod2usage({
-message => 'ERROR: Connection parameters (url or reg_conf+reg_alias) need to be specified',
-exitvalue => 1,
-verbose => 1
});
}
if(! $self->{output}) {
pod2usage({
-message => 'ERROR: No -output flag given',
-exitvalue => 1,
-verbose => 1
});
}
if(! $self->{output}) {
pod2usage({
-message => 'ERROR: No -output flag given',
-exitvalue => 1,
-verbose => 1
});
}
if(!$self->{format}) {
if($self->{output}=~/\.(\w+)$/) {
$self->{format} = $1;
} else {
die "Format was not set and could not guess from ".$self->output().". Please use either way to select it.\n";
if(!$self->{format}) {
if($self->{output}=~/\.(\w+)$/) {
$self->{format} = $1;
} else {
die "Format was not set and could not guess from ".$self->output().". Please use either way to select it.\n";
}
}
}
}
sub _write_graph {
my ($self) = @_;
my $graph = Bio::EnsEMBL::Hive::Utils::Graph->new( $self->{dba} );
my $graphviz = $graph->build();
my $call = q{as_}.$self->{format};
eval {$graphviz->$call($self->{output});};
if($@) {
warn $@;
pod2usage({
-message => 'Error detected. Check '.$self->{format}.' is a valid format. Use a format name as supported by graphviz',
-exitvalue => 1,
-verbose => 1
});
}
my ($self) = @_;
my $graph = Bio::EnsEMBL::Hive::Utils::Graph->new( $self->{dba} );
my $graphviz = $graph->build();
my $call = q{as_}.$self->{format};
eval {$graphviz->$call($self->{output});};
if($@) {
warn $@;
pod2usage({
-message => 'Error detected. Check '.$self->{format}.' is a valid format. Use a format name as supported by graphviz',
-exitvalue => 1,
-verbose => 1
});
}
}
__END__
=pod
=head1 NAME
......@@ -136,51 +127,31 @@ hive_config.json configuration file.
=item B<--format>
The format of the file output. See FORMATS for more information
The format of the file output. See FORMATS for more information
=item B<--output>
Location of the file to write to.
Location of the file to write to.
=item B<-reg_conf>
path to a Registry configuration file
path to a Registry configuration file
=item B<-reg_alias>
species/alias name for the Hive DBAdaptor
species/alias name for the Hive DBAdaptor
=item B<-url>
url defining where hive database is located
=item B<-host>
mysql database host <machine>
=item B<-port>
mysql port number
=item B<-user>
mysql connection user <name>
=item B<-password>
mysql connection password <pass>
=item B<-database>
mysql database <name>
url defining where hive database is located
=back
=head1 FORMATS
The script supports the same output formats as GraphViz & the accompanying
Perl module do. However here are a list of common output formats you may
want to specify (png is the default).
The script supports the same output formats as GraphViz & the accompanying
Perl module do. However here are a list of common output formats you may
want to specify (png is the default).
=over 8
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment