long_mult_semaphores.txt 2.47 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
############################################################################################################################
#
# Please see the long_mult_example_pipeline.txt first.
#
# This is an example ( a follow-up on 'long_mult_example_pipeline.txt', so make sure you have read it first)
# of how to set up a pipeline with counting semaphores.
#
############################################################################################################################

# 0. Cache MySQL connection parameters in a variable (they will work as eHive connection parameters as well) :
export MYCONN="--host=hostname --port=port_number --user=username --password=secret"
#
# also, set the ENS_CODE_ROOT to the directory where ensembl packages are installed:
export ENS_CODE_ROOT="$HOME/ensembl_main"

# 1. Create an empty database:
mysql $MYCONN -e 'DROP DATABASE IF EXISTS long_mult_test'
mysql $MYCONN -e 'CREATE DATABASE long_mult_test'

# 2. Create eHive infrastructure:
mysql $MYCONN long_mult_test <$ENS_CODE_ROOT/ensembl-hive/sql/tables.sql

# 3. Create analyses/control_rules/dataflow_rules of the LongMult pipeline:
mysql $MYCONN long_mult_test <$ENS_CODE_ROOT/ensembl-hive/sql/create_sema_long_mult.sql

# 4. "Load" the pipeline with a multiplication task:
mysql $MYCONN long_mult_test <<EoF
INSERT INTO analysis_job (analysis_id, input_id) VALUES ( 1, "{ 'a_multiplier' => '9650516169', 'b_multiplier' => '327358788' }");
INSERT INTO analysis_job (analysis_id, input_id) VALUES ( 1, "{ 'a_multiplier' => '327358788', 'b_multiplier' => '9650516169' }");
EoF

# 5. Initialize the newly created eHive for the first time:
beekeeper.pl $MYCONN --database=long_mult_test -sync

# 6. You can either execute three individual workers (each picking one analysis of the pipeline):
runWorker.pl $MYCONN --database=long_mult_test
#
# ... or run an automatic loop that will run workers for you:
beekeeper.pl $MYCONN --database=long_mult_test -loop
#
# KNOWN BUG: if you keep suggesting your own analysis_id/logic_name, the system may sometimes think there is no work,
#            where actually there will be some previously semaphored jobs that have become available yet invisible to some workers.
# KNOWN FIX: just run "beekeeper.pl $MYCONN --database=long_mult_test -sync" once, and the problem should rectify itself.

# 7. The results of the computations are to be found in 'final_result' table:
mysql $MYCONN long_mult_test -e 'SELECT * FROM final_result'

# 8. You can add more multiplication tasks by repeating from step 4.