Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
ensembl-gh-mirror
ensembl-hive
Commits
c665fbfc
Commit
c665fbfc
authored
Aug 24, 2012
by
Leo Gordon
Browse files
inherit Job,Worker,DFR,RC from Bio::EnsEMBL::Storable, reuse some code
parent
cb325115
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
71 additions
and
132 deletions
+71
-132
modules/Bio/EnsEMBL/Hive/AnalysisJob.pm
modules/Bio/EnsEMBL/Hive/AnalysisJob.pm
+11
-25
modules/Bio/EnsEMBL/Hive/DataflowRule.pm
modules/Bio/EnsEMBL/Hive/DataflowRule.pm
+11
-40
modules/Bio/EnsEMBL/Hive/Process.pm
modules/Bio/EnsEMBL/Hive/Process.pm
+1
-1
modules/Bio/EnsEMBL/Hive/Queen.pm
modules/Bio/EnsEMBL/Hive/Queen.pm
+6
-4
modules/Bio/EnsEMBL/Hive/ResourceClass.pm
modules/Bio/EnsEMBL/Hive/ResourceClass.pm
+7
-30
modules/Bio/EnsEMBL/Hive/Worker.pm
modules/Bio/EnsEMBL/Hive/Worker.pm
+35
-32
No files found.
modules/Bio/EnsEMBL/Hive/AnalysisJob.pm
View file @
c665fbfc
...
...
@@ -27,23 +27,24 @@
package
Bio::EnsEMBL::Hive::
AnalysisJob
;
use
strict
;
use
Scalar::
Util
('
weaken
');
use
Bio::EnsEMBL::Utils::
Argument
;
# import 'rearrange()'
use
Bio::EnsEMBL::Hive::DBSQL::
AnalysisJobAdaptor
;
use
Bio::EnsEMBL::Hive::DBSQL::
DataflowRuleAdaptor
;
use
base
('
Bio::EnsEMBL::Hive::Params
');
use
base
(
'
Bio::EnsEMBL::Storable
',
# inherit dbID(), adaptor() and new() methods
'
Bio::EnsEMBL::Hive::Params
',
# inherit param management functionality
);
sub
new
{
my
$class
=
shift
@_
;
my
$self
=
bless
{},
$class
;
my
$self
=
$class
->
SUPER::
new
(
@
_
);
# deal with Storable stuff
my
(
$dbID
,
$analysis_id
,
$input_id
,
$worker_id
,
$status
,
$retry_count
,
$completed
,
$runtime_msec
,
$query_count
,
$semaphore_count
,
$semaphored_job_id
,
$adaptor
)
=
rearrange
([
qw(
dbID
analysis_id input_id worker_id status retry_count completed runtime_msec query_count semaphore_count semaphored_job_id
adaptor
)
],
@
_
);
my
(
$analysis_id
,
$input_id
,
$worker_id
,
$status
,
$retry_count
,
$completed
,
$runtime_msec
,
$query_count
,
$semaphore_count
,
$semaphored_job_id
)
=
rearrange
([
qw(analysis_id input_id worker_id status retry_count completed runtime_msec query_count semaphore_count semaphored_job_id)
],
@
_
);
$self
->
dbID
(
$dbID
)
if
(
defined
(
$dbID
));
$self
->
analysis_id
(
$analysis_id
)
if
(
defined
(
$analysis_id
));
$self
->
input_id
(
$input_id
)
if
(
defined
(
$input_id
));
$self
->
worker_id
(
$worker_id
)
if
(
defined
(
$worker_id
));
...
...
@@ -54,28 +55,10 @@ sub new {
$self
->
query_count
(
$query_count
)
if
(
defined
(
$query_count
));
$self
->
semaphore_count
(
$semaphore_count
)
if
(
defined
(
$semaphore_count
));
$self
->
semaphored_job_id
(
$semaphored_job_id
)
if
(
defined
(
$semaphored_job_id
));
$self
->
adaptor
(
$adaptor
)
if
(
defined
(
$adaptor
));
return
$self
;
}
sub
adaptor
{
my
$self
=
shift
@_
;
if
(
@
_
)
{
$self
->
{'
_adaptor
'}
=
shift
@_
;
weaken
$self
->
{'
_adaptor
'};
}
return
$self
->
{'
_adaptor
'};
}
sub
dbID
{
my
$self
=
shift
;
$self
->
{'
_dbID
'}
=
shift
if
(
@
_
);
return
$self
->
{'
_dbID
'};
}
sub
input_id
{
my
$self
=
shift
;
...
...
@@ -83,6 +66,7 @@ sub input_id {
return
$self
->
{'
_input_id
'};
}
sub
dataflow_rules
{
# if ever set will prevent the Job from fetching rules from the DB
my
$self
=
shift
@_
;
my
$branch_name_or_code
=
shift
@_
;
...
...
@@ -96,12 +80,14 @@ sub dataflow_rules { # if ever set will prevent the Job from fetching rules f
:
$self
->
adaptor
->
db
->
get_DataflowRuleAdaptor
->
fetch_all_by_from_analysis_id_and_branch_code
(
$self
->
analysis_id
,
$branch_code
);
}
sub
worker_id
{
my
$self
=
shift
;
$self
->
{'
_worker_id
'}
=
shift
if
(
@
_
);
return
$self
->
{'
_worker_id
'};
}
sub
analysis_id
{
my
$self
=
shift
;
$self
->
{'
_analysis_id
'}
=
shift
if
(
@
_
);
...
...
modules/Bio/EnsEMBL/Hive/DataflowRule.pm
View file @
c665fbfc
...
...
@@ -50,13 +50,16 @@
package
Bio::EnsEMBL::Hive::
DataflowRule
;
use
strict
;
use
Scalar::
Util
('
weaken
');
use
Bio::EnsEMBL::Utils::
Argument
;
# import 'rearrange()'
use
Bio::EnsEMBL::Utils::
Exception
;
use
Bio::EnsEMBL::Hive::
Utils
('
stringify
');
# import 'stringify()'
use
Bio::EnsEMBL::Hive::DBSQL::
AnalysisAdaptor
;
use
base
(
'
Bio::EnsEMBL::Storable
',
# inherit dbID(), adaptor() and new() methods
);
=head2 new
Usage : Bio::EnsEMBL::Hive::DataflowRule->new(-from_analysis => $fromAnalysis, -to_analysis => $toAnalysis, -branch_code => $branch_code);
...
...
@@ -68,14 +71,11 @@ use Bio::EnsEMBL::Hive::DBSQL::AnalysisAdaptor;
sub
new
{
my
$class
=
shift
@_
;
my
$self
=
bless
{},
$class
;
my
(
$dbID
,
$adaptor
,
$fromAnalysis
,
$toAnalysis
,
$from_analysis_id
,
$branch_code
,
$funnel_dataflow_rule_id
,
$to_analysis_url
,
$input_id_template
)
=
rearrange
(
[
qw (DBID
ADAPTOR
FROM_ANALYSIS
TO_ANALYSIS
FROM_ANALYSIS_ID
BRANCH_CODE
FUNNEL_DATAFLOW_RULE_ID
TO_ANALYSIS_URL
INPUT_ID_TEMPLATE
)
],
@
_
);
my
$self
=
$class
->
SUPER::
new
(
@
_
);
# deal with Storable stuff
# database persistence:
$self
->
dbID
(
$dbID
)
if
(
defined
(
$dbID
));
$self
->
adaptor
(
$adaptor
)
if
(
defined
(
$adaptor
));
my
(
$fromAnalysis
,
$toAnalysis
,
$from_analysis_id
,
$branch_code
,
$funnel_dataflow_rule_id
,
$to_analysis_url
,
$input_id_template
)
=
rearrange
(
[
qw (FROM_ANALYSIS
TO_ANALYSIS
FROM_ANALYSIS_ID
BRANCH_CODE
FUNNEL_DATAFLOW_RULE_ID
TO_ANALYSIS_URL
INPUT_ID_TEMPLATE
)
],
@
_
);
# from objects:
$self
->
from_analysis
(
$fromAnalysis
)
if
(
defined
(
$fromAnalysis
));
...
...
@@ -91,38 +91,6 @@ sub new {
return
$self
;
}
=head2 dbID
Function: getter/setter method for the dbID of the dataflow rule
=cut
sub
dbID
{
my
$self
=
shift
@_
;
if
(
@
_
)
{
# setter mode
$self
->
{'
_dbID
'}
=
shift
@_
;
}
return
$self
->
{'
_dbID
'};
}
=head2 adaptor
Function: getter/setter method for the adaptor of the dataflow rule
=cut
sub
adaptor
{
my
$self
=
shift
@_
;
if
(
@
_
)
{
$self
->
{'
_adaptor
'}
=
shift
@_
;
weaken
$self
->
{'
_adaptor
'};
}
return
$self
->
{'
_adaptor
'};
}
=head2 branch_code
...
...
@@ -139,6 +107,7 @@ sub branch_code {
return
$self
->
{'
_branch_code
'};
}
=head2 funnel_dataflow_rule_id
Function: getter/setter method for the funnel_dataflow_rule_id of the dataflow rule
...
...
@@ -154,6 +123,7 @@ sub funnel_dataflow_rule_id {
return
$self
->
{'
_funnel_dataflow_rule_id
'};
}
=head2 input_id_template
Function: getter/setter method for the input_id_template of the dataflow rule
...
...
@@ -170,6 +140,7 @@ sub input_id_template {
return
$self
->
{'
_input_id_template
'};
}
=head2 from_analysis_id
Arg[1] : (optional) int $dbID
...
...
modules/Bio/EnsEMBL/Hive/Process.pm
View file @
c665fbfc
...
...
@@ -175,7 +175,7 @@ sub enter_status {
}
if
(
$worker
)
{
$worker
->
status
(
$status
);
$worker
->
queen
->
check_in_worker
(
$worker
);
$worker
->
adaptor
->
check_in_worker
(
$worker
);
}
}
...
...
modules/Bio/EnsEMBL/Hive/Queen.pm
View file @
c665fbfc
...
...
@@ -212,6 +212,8 @@ sub create_new_worker {
my
$worker
=
$self
->
fetch_by_dbID
(
$worker_id
);
$worker
=
undef
unless
(
$worker
and
$worker
->
analysis
);
$worker
->
init
;
if
(
$worker
and
$analysisStats
)
{
$analysisStats
->
update_status
('
WORKING
');
}
...
...
@@ -1060,10 +1062,11 @@ sub _objs_from_sth {
my
@workers
=
();
while
(
$sth
->
fetch
())
{
my
$worker
=
Bio::EnsEMBL::Hive::
Worker
->
new
();
$worker
->
init
;
my
$worker
=
Bio::EnsEMBL::Hive::
Worker
->
new
(
# will be passed to Storable's new()
-
adaptor
=>
$self
,
-
dbID
=>
$column
{'
worker_id
'},
);
$worker
->
dbID
(
$column
{'
worker_id
'});
$worker
->
meadow_type
(
$column
{'
meadow_type
'});
$worker
->
meadow_name
(
$column
{'
meadow_name
'});
$worker
->
host
(
$column
{'
host
'});
...
...
@@ -1074,7 +1077,6 @@ sub _objs_from_sth {
$worker
->
last_check_in
(
$column
{'
last_check_in
'});
$worker
->
died
(
$column
{'
died
'});
$worker
->
cause_of_death
(
$column
{'
cause_of_death
'});
$worker
->
queen
(
$self
);
$worker
->
db
(
$self
->
db
);
if
(
$column
{'
analysis_id
'}
and
$self
->
db
->
get_AnalysisAdaptor
)
{
...
...
modules/Bio/EnsEMBL/Hive/ResourceClass.pm
View file @
c665fbfc
...
...
@@ -26,49 +26,26 @@
package
Bio::EnsEMBL::Hive::
ResourceClass
;
use
strict
;
use
Scalar::
Util
('
weaken
');
use
Bio::EnsEMBL::Utils::
Argument
;
# import 'rearrange()'
use
base
(
'
Bio::EnsEMBL::Storable
',
# inherit dbID(), adaptor() and new() methods
);
sub
new
{
my
$class
=
shift
@_
;
my
$self
=
bless
{},
$class
;
my
$self
=
$class
->
SUPER::
new
(
@
_
);
# deal with Storable stuff
my
(
$adaptor
,
$dbID
,
$name
)
=
rearrange
([
qw(
adaptor dbID
name)
],
@
_
);
my
(
$name
)
=
rearrange
([
qw(name)
],
@
_
);
$self
->
adaptor
(
$adaptor
)
if
(
defined
(
$adaptor
));
$self
->
dbID
(
$dbID
);
$self
->
name
(
$name
);
$self
->
name
(
$name
)
if
(
$name
);
return
$self
;
}
sub
adaptor
{
my
$self
=
shift
@_
;
if
(
@
_
)
{
$self
->
{'
_adaptor
'}
=
shift
@_
;
weaken
$self
->
{'
_adaptor
'};
}
return
$self
->
{'
_adaptor
'};
}
sub
dbID
{
my
$self
=
shift
@_
;
if
(
@
_
)
{
$self
->
{'
_resource_class_id
'}
=
shift
@_
;
}
return
$self
->
{'
_resource_class_id
'};
}
sub
name
{
my
$self
=
shift
@_
;
...
...
modules/Bio/EnsEMBL/Hive/Worker.pm
View file @
c665fbfc
...
...
@@ -64,67 +64,75 @@
package
Bio::EnsEMBL::Hive::
Worker
;
use
strict
;
use
Bio::EnsEMBL::Utils::
Argument
;
use
Bio::EnsEMBL::Utils::
Exception
;
use
Bio::EnsEMBL::Hive::Utils::
Stopwatch
;
use
POSIX
;
use
Bio::EnsEMBL::
Analysis
;
use
Bio::EnsEMBL::DBSQL::
DBAdaptor
;
use
Bio::EnsEMBL::Utils::
Argument
;
use
Bio::EnsEMBL::Utils::
Exception
;
use
Bio::EnsEMBL::Hive::
Extensions
;
use
Bio::EnsEMBL::Hive::
Process
;
use
Bio::EnsEMBL::Hive::Utils::
Stopwatch
;
use
Bio::EnsEMBL::Hive::DBSQL::
AnalysisJobAdaptor
;
use
Bio::EnsEMBL::Hive::DBSQL::
AnalysisStatsAdaptor
;
use
Bio::EnsEMBL::Hive::DBSQL::
DataflowRuleAdaptor
;
use
Bio::EnsEMBL::Hive::
Extensions
;
use
Bio::EnsEMBL::Hive::
Process
;
use
Bio::EnsEMBL::Hive::Utils::
RedirectStack
;
use
Bio::EnsEMBL::Hive::
Utils
('
dir_revhash
');
# import dir_revhash
use
base
(
'
Bio::EnsEMBL::Storable
',
# inherit dbID(), adaptor() and new() methods
);
sub
new
{
my
(
$class
,
@args
)
=
@_
;
my
$self
=
bless
{},
$class
;
return
$self
;
my
$class
=
shift
@_
;
my
$self
=
$class
->
SUPER::
new
(
@
_
);
# deal with Storable stuff
return
$self
;
}
sub
init
{
my
$self
=
shift
;
my
$self
=
shift
;
my
$lifespan_stopwatch
=
Bio::EnsEMBL::Hive::Utils::
Stopwatch
->
new
();
$lifespan_stopwatch
->
_unit
(
1
);
# count in seconds (default is milliseconds)
$lifespan_stopwatch
->
restart
;
$self
->
lifespan_stopwatch
(
$lifespan_stopwatch
);
my
$lifespan_stopwatch
=
Bio::EnsEMBL::Hive::Utils::
Stopwatch
->
new
();
$lifespan_stopwatch
->
_unit
(
1
);
# count in seconds (default is milliseconds)
$lifespan_stopwatch
->
restart
;
$self
->
lifespan_stopwatch
(
$lifespan_stopwatch
);
$self
->
debug
(
0
);
return
$self
;
return
$self
;
}
sub
queen
{
my
$self
=
shift
;
$self
->
{'
_queen
'}
=
shift
if
(
@
_
);
return
$self
->
{'
_queen
'};
}
sub
db
{
my
$self
=
shift
;
$self
->
{'
_db
'}
=
shift
if
(
@
_
);
return
$self
->
{'
_db
'};
}
sub
meadow_type
{
my
$self
=
shift
;
$self
->
{'
_meadow_type
'}
=
shift
if
(
@
_
);
return
$self
->
{'
_meadow_type
'};
}
sub
meadow_name
{
my
$self
=
shift
;
$self
->
{'
_meadow_name
'}
=
shift
if
(
@
_
);
return
$self
->
{'
_meadow_name
'};
}
sub
debug
{
my
$self
=
shift
;
$self
->
{'
_debug
'}
=
shift
if
(
@
_
);
$self
->
{'
_debug
'}
=
0
unless
(
defined
(
$self
->
{'
_debug
'}));
return
$self
->
{'
_debug
'};
}
sub
execute_writes
{
my
$self
=
shift
;
$self
->
{'
_execute_writes
'}
=
shift
if
(
@
_
);
...
...
@@ -132,6 +140,7 @@ sub execute_writes {
return
$self
->
{'
_execute_writes
'};
}
=head2 analysis
Arg [1] : (optional) Bio::EnsEMBL::Analysis $value
...
...
@@ -284,12 +293,6 @@ sub prev_job_error {
}
sub
dbID
{
my
(
$self
,
$value
)
=
@_
;
$self
->
{'
_worker_id
'}
=
$value
if
(
$value
);
return
$self
->
{'
_worker_id
'};
}
sub
host
{
my
(
$self
,
$value
)
=
@_
;
$self
->
{'
_host
'}
=
$value
if
(
$value
);
...
...
@@ -583,7 +586,7 @@ sub run {
}
}
$self
->
queen
->
register_worker_death
(
$self
);
$self
->
adaptor
->
register_worker_death
(
$self
);
$self
->
analysis
->
stats
->
print_stats
if
(
$self
->
debug
);
...
...
@@ -603,8 +606,8 @@ sub run_one_batch {
my
$max_retry_count
=
$self
->
analysis
->
stats
->
max_retry_count
();
# a constant (as the Worker is already specialized by the Queen) needed later for retrying jobs
$self
->
queen
->
check_in_worker
(
$self
);
$self
->
queen
->
safe_synchronize_AnalysisStats
(
$self
->
analysis
->
stats
);
$self
->
adaptor
->
check_in_worker
(
$self
);
$self
->
adaptor
->
safe_synchronize_AnalysisStats
(
$self
->
analysis
->
stats
);
$self
->
cause_of_death
('
NO_WORK
')
unless
(
scalar
@
{
$jobs
});
...
...
@@ -715,7 +718,7 @@ sub enter_status {
$job
->
update_status
(
$status
);
}
$self
->
status
(
$status
);
$self
->
queen
->
check_in_worker
(
$self
);
$self
->
adaptor
->
check_in_worker
(
$self
);
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment