Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
ensembl-gh-mirror
ensembl-hive
Commits
5bdd25c8
Commit
5bdd25c8
authored
Feb 23, 2010
by
Leo Gordon
Browse files
process_id code moved to Meadow, bk detection makes setting this option useless
parent
014ee4f4
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
56 additions
and
32 deletions
+56
-32
modules/Bio/EnsEMBL/Hive/Meadow/LOCAL.pm
modules/Bio/EnsEMBL/Hive/Meadow/LOCAL.pm
+7
-1
modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm
modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm
+17
-0
scripts/runWorker.pl
scripts/runWorker.pl
+32
-31
No files found.
modules/Bio/EnsEMBL/Hive/Meadow/LOCAL.pm
View file @
5bdd25c8
...
...
@@ -7,6 +7,12 @@ use Sys::Hostname;
use
base
'
Bio::EnsEMBL::Hive::Meadow
';
sub
get_current_worker_process_id
{
my
(
$self
)
=
@_
;
return
$$
;
}
sub
count_running_workers
{
my
$self
=
shift
@_
;
...
...
@@ -20,7 +26,7 @@ sub count_running_workers {
sub
responsible_for_worker
{
my
(
$self
,
$worker
)
=
@_
;
return
(
$worker
->
beekeeper
()
eq
$self
->
type
()
)
&&
(
$worker
->
host
eq
hostname
());
return
(
$self
->
SUPER::
responsible_for_worker
(
$worker
)
&&
(
$worker
->
host
eq
hostname
())
)
;
}
sub
status_of_all_my_workers
{
# returns a hashref
...
...
modules/Bio/EnsEMBL/Hive/Meadow/LSF.pm
View file @
5bdd25c8
...
...
@@ -6,6 +6,23 @@ use strict;
use
base
'
Bio::EnsEMBL::Hive::Meadow
';
sub
get_current_worker_process_id
{
my
(
$self
)
=
@_
;
my
$lsb_jobid
=
$ENV
{'
LSB_JOBID
'};
my
$lsb_jobindex
=
$ENV
{'
LSB_JOBINDEX
'};
if
(
defined
(
$lsb_jobid
)
and
defined
(
$lsb_jobindex
))
{
if
(
$lsb_jobindex
>
0
)
{
return
"
$lsb_jobid
\
[
$lsb_jobindex
\
]
";
}
else
{
return
$lsb_jobid
;
}
}
else
{
die
"
Could not establish the process_id
";
}
}
sub
count_pending_workers
{
my
(
$self
)
=
@_
;
...
...
scripts/runWorker.pl
View file @
5bdd25c8
...
...
@@ -9,6 +9,9 @@ use Bio::EnsEMBL::Hive::Worker;
use
Bio::EnsEMBL::Hive::
Queen
;
use
Bio::EnsEMBL::
Registry
;
use
Bio::EnsEMBL::Hive::Meadow::
LSF
;
use
Bio::EnsEMBL::Hive::Meadow::
LOCAL
;
Bio::EnsEMBL::
Registry
->
no_version_check
(
1
);
# ok this is a hack, but I'm going to pretend I've got an object here
...
...
@@ -58,7 +61,7 @@ GetOptions(
'
limit=i
'
=>
\
$self
->
{'
job_limit
'},
'
lifespan=i
'
=>
\
$self
->
{'
lifespan
'},
'
outdir=s
'
=>
\
$self
->
{'
outdir
'},
'
bk=s
'
=>
\
$self
->
{'
beekeeper
'},
'
bk=s
'
=>
\
$self
->
{'
beekeeper
'},
# deprecated and ignored
'
pid=s
'
=>
\
$self
->
{'
process_id
'},
'
input_id=s
'
=>
\
$self
->
{'
input_id
'},
'
no_cleanup
'
=>
\
$self
->
{'
no_global_cleanup
'},
...
...
@@ -113,29 +116,24 @@ unless($DBA and $DBA->isa("Bio::EnsEMBL::Hive::DBSQL::DBAdaptor")) {
my
$queen
=
$DBA
->
get_Queen
();
$queen
->
{
maximise_concurrency
}
=
1
if
(
$self
->
{
maximise_concurrency
});
################################
# LSF submit system dependency
# no nice way to move this outside, so inside here.
# environment variables LSB_JOBID and LSB_JOBINDEX are set for process started
# by LSF deamon. Also know that the beekeeper is 'LSF'
#
my
$lsb_jobid
=
$ENV
{'
LSB_JOBID
'};
my
$lsb_jobindex
=
$ENV
{'
LSB_JOBINDEX
'};
if
(
defined
(
$lsb_jobid
)
and
defined
(
$lsb_jobindex
))
{
$self
->
{'
beekeeper
'}
=
'
LSF
'
unless
(
$self
->
{'
beekeeper
'});
if
(
$lsb_jobindex
>
0
)
{
$self
->
{'
process_id
'}
=
"
$lsb_jobid
\
[
$lsb_jobindex
\
]
";
}
else
{
$self
->
{'
process_id
'}
=
"
$lsb_jobid
";
}
unless
(
$self
->
{'
process_id
'})
{
# do we really need this confusing feature - to be able to set the process_id externally?
eval
{
$self
->
{'
process_id
'}
=
Bio::EnsEMBL::Hive::Meadow::
LSF
->
get_current_worker_process_id
();
};
if
(
$@
)
{
$self
->
{'
process_id
'}
=
Bio::EnsEMBL::Hive::Meadow::
LOCAL
->
get_current_worker_process_id
();
$self
->
{'
beekeeper
'}
=
'
LOCAL
';
}
else
{
$self
->
{'
beekeeper
'}
=
'
LSF
';
}
}
################################
print
("
pid =
",
$self
->
{'
process_id
'},
"
\n
")
if
(
$self
->
{'
process_id
'});
if
(
$self
->
{'
logic_name
'})
{
my
$analysis
=
$queen
->
db
->
get_AnalysisAdaptor
->
fetch_by_logic_name
(
$self
->
{'
logic_name
'});
unless
(
$analysis
)
{
printf
("
logic_name:'%s' does not exist in database
\n\n
",
$self
->
{'
logic_name
'});
printf
("
logic_name:
'%s' does not exist in database
\n\n
",
$self
->
{'
logic_name
'});
usage
(
1
);
}
$self
->
{'
analysis_id
'}
=
$analysis
->
dbID
;
...
...
@@ -168,7 +166,7 @@ my $worker = $queen->create_new_worker(
unless
(
$worker
)
{
$queen
->
print_analysis_status
if
(
$self
->
{'
show_analysis_stats
'});
print
("
\n
=== COULDN'T CREATE WORKER ===
\n
");
exit
(
0
);
exit
(
1
);
}
$worker
->
debug
(
$self
->
{'
debug
'})
if
(
$self
->
{'
debug
'});
...
...
@@ -197,28 +195,31 @@ if($self->{'no_global_cleanup'}) {
$worker
->
print_worker
();
my
$return_value
=
0
;
eval
{
$worker
->
run
();
};
if
(
$@
)
{
#worker threw an exception so it had a problem
if
(
$worker
->
perform_global_cleanup
)
{
#have runnable cleanup any global/process files/data it may have created
$worker
->
cleanup_worker_process_temp_directory
;
}
print
("
\n
$@
");
$queen
->
register_worker_death
(
$worker
);
# try to capture it ASAP:
$return_value
=
(
$!
||
$?
>>
8
||
1
);
#worker threw an exception so it had a problem:
if
(
$worker
->
perform_global_cleanup
)
{
#have runnable cleanup any global/process files/data it may have created
$worker
->
cleanup_worker_process_temp_directory
;
}
print
("
\n
$@
");
$queen
->
register_worker_death
(
$worker
);
}
if
(
$self
->
{'
show_analysis_stats
'})
{
$queen
->
print_analysis_status
;
$queen
->
get_num_needed_workers
();
$queen
->
print_analysis_status
;
$queen
->
get_num_needed_workers
();
}
printf
("
dbc %d disconnect cycles
\n
",
$DBA
->
dbc
->
disconnect_count
);
print
("
total jobs completes :
",
$worker
->
work_done
,
"
\n
");
exit
(
0
);
exit
(
$return_value
);
#######################
...
...
@@ -312,7 +313,7 @@ runWorker.pl -url mysql://username:secret@hostname:port/ehive_dbname -logic_name
-limit <num> : #jobs to run before worker can die naturally
-lifespan <num> : number of minutes this worker is allowed to run
-outdir <path> : directory where stdout/stderr is redirected
-bk <string> : beekeeper identifier
-bk <string> : beekeeper identifier
(deprecated and ignored)
-pid <string> : externally set process_id descriptor (e.g. lsf job_id, array_id)
-input_id <string> : test input_id on specified analysis (analysis_id or logic_name)
-job_id <id> : run specific job defined by analysis_job_id
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment