Updated the script documentation for 2.1

docs/index.html

@@ -31,7 +31,7 @@ The name "Hive" comes from the way pipelines are processed by a swarm
 	<span class="tree">├── </span><a href="scripts/hoover_pipeline.html">hoover_pipeline</a><br>
 	<span class="tree">│   </span><em>Reporting</em><br>
 	<span class="tree">├── </span><a href="scripts/generate_graph.html">generate_graph</a><br>
-	<span class="tree">├── </span><a href="scripts/lsf_report.html">lsf_report</a><br>
+	<span class="tree">├── </span><a href="scripts/load_resource_usage.html">load_resource_usage</a><br>
 	<span class="tree">└── </span><a href="scripts/generate_timeline.html">generate_timeline</a><br>
 </div>
 </li>

docs/scripts/beekeeper.html

@@ -56,6 +56,10 @@
     -reg_alias <string>    : species/alias name for the Hive DBAdaptor
     -url &lt;url string&gt;      : url defining where hive database is located</code></pre>
 
+<h2 id="Configs-overriding">Configs overriding</h2>
+
+<pre><code>    -config_file &lt;string&gt;  : json file (with absolute path) to override the default configurations (could be multiple)</code></pre>
+
 <h2 id="Looping-control">Looping control</h2>
 
 <pre><code>    -loop                  : run autonomously, loops and sleeps
@@ -63,7 +67,7 @@
     -keep_alive            : do not stop when there are no more jobs to do - carry on looping
     -job_id &lt;job_id&gt;       : run 1 iteration for this job_id
     -run                   : run 1 iteration of automation loop
-    -sleep &lt;num&gt;           : when looping, sleep &lt;num&gt; minutes (default 2min)</code></pre>
+    -sleep &lt;num&gt;           : when looping, sleep &lt;num&gt; minutes (default 1 min)</code></pre>
 
 <h2 id="Current-Meadow-control">Current Meadow control</h2>
 

docs/scripts/generate_graph.html

@@ -17,7 +17,7 @@
 
 <h1 id="SYNOPSIS">SYNOPSIS</h1>
 
-<pre><code>    ./generate_graph.pl -url mysql://user:pass@server:port/dbname -output OUTPUT_LOC [-help]</code></pre>
+<pre><code>    ./generate_graph.pl [ -url mysql://user:pass@server:port/dbname ] [-pipeconfig TopUp_conf.pm]* -output OUTPUT_LOC [-help]</code></pre>
 
 <h1 id="DESCRIPTION">DESCRIPTION</h1>
 
@@ -41,6 +41,11 @@
 
 <pre><code>    species/alias name for the Hive DBAdaptor</code></pre>
 
+<p><b>--pipeconfig</b></p>
+
+<pre><code>    A pipeline configuration file that can function both as the initial source of pipeline structure or as a top-up config.
+    This option can now be used multiple times for multiple top-ups.</code></pre>
+
 <p><b>--output</b></p>
 
 <pre><code>    Location of the file to write to.

docs/scripts/init_pipeline.html

@@ -17,7 +17,7 @@
 
 <h1 id="SYNOPSIS">SYNOPSIS</h1>
 
-<pre><code>    init_pipeline.pl &lt;config_module_or_filename&gt; [-help | [-analysis_topup | -job_topup] &lt;options_for_this_particular_pipeline&gt;]</code></pre>
+<pre><code>    init_pipeline.pl &lt;config_module_or_filename&gt; [&lt;options_for_this_particular_pipeline&gt;]</code></pre>
 
 <h1 id="DESCRIPTION">DESCRIPTION</h1>
 
@@ -31,24 +31,13 @@
         # initialize a generic eHive pipeline:
     init_pipeline.pl Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf -password &lt;yourpassword&gt;
 
-        # see what command line options are available when initializing long multiplication example pipeline
-        #   (assuming your current directory is ensembl-hive/modules/Bio/EnsEMBL/Hive) :
-    init_pipeline.pl PipeConfig/LongMult_conf -help
-
         # initialize the long multiplicaton pipeline by supplying not only mandatory but also optional data:
         #   (assuming your current directory is ensembl-hive/modules/Bio/EnsEMBL/Hive/PipeConfig) :
     init_pipeline.pl LongMult_conf -password &lt;yourpassword&gt; -first_mult 375857335 -second_mult 1111333355556666 </code></pre>
 
 <h1 id="OPTIONS">OPTIONS</h1>
 
-<pre><code>    -help            :   Gets this help message and exits
-
-    -analysis_topup  :   A special initialization mode when (1) pipeline_create_commands are switched off and (2) only newly defined analyses are added to the database
-                         This mode is only useful in the process of putting together a new pipeline.
-
-    -job_topup       :   Another special initialization mode when only jobs are created - no other structural changes to the pipeline are acted upon.
-
-    -hive_force_init :   If set to 1, forces the (re)creation of the hive database even if a previous version of it is present in the server.</code></pre>
+<pre><code>    -hive_force_init :   If set to 1, forces the (re)creation of the hive database even if a previous version of it is present in the server.</code></pre>
 
 <h1 id="LICENSE">LICENSE</h1>
 

docs/scripts/lsf_report.html → docs/scripts/load_resource_usage.html

@@ -2,7 +2,7 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-<title>lsf_report.pl</title>
+<title>load_resource_usage.pl</title>
 <meta http-equiv="content-type" content="text/html; charset=utf-8" />
 <link rev="made" href="mailto:root@localhost" />
 </head>
@@ -13,47 +13,41 @@
 
 <h1 id="NAME">NAME</h1>
 
-<pre><code>    lsf_report.pl</code></pre>
+<pre><code>    load_resource_usage.pl</code></pre>
 
 <h1 id="DESCRIPTION">DESCRIPTION</h1>
 
-<pre><code>    This script is used for offline examination of resources used by a Hive pipeline running on LSF
-    (the script is [Pp]latform-dependent).
+<pre><code>    This script obtains resource usage data for your pipeline from the Meadow and stores it in &#39;worker_resource_usage&#39; table.
+    Your Meadow class/plugin has to support offline examination of resources in order for this script to work.
 
-    Based on the command-line parameters &#39;start_date&#39; and &#39;end_date&#39;, or on the start time of the first
-    worker and end time of the last worker (as recorded in pipeline DB), it pulls the relevant data out
-    of LSF&#39;s &#39;bacct&#39; database, parses it and stores in &#39;lsf_report&#39; table.
+    Based on the start time of the first Worker and end time of the last Worker (as recorded in pipeline DB),
+    it pulls the relevant data out of your Meadow (runs &#39;bacct&#39; script in case of LSF), parses the report and stores in &#39;worker_resource_usage&#39; table.
     You can join this table to &#39;worker&#39; table USING(meadow_name,process_id) in the usual MySQL way
     to filter by analysis_id, do various stats, etc.
 
-    You can optionally ask the script to dump the &#39;bacct&#39; database in a dump file,
-    or fill in the &#39;lsf_report&#39; table from an existing dump file (most time is taken by querying bacct).
-
-    Please note the script may additionally pull information about LSF processes that you ran simultaneously
-    with running the pipeline. It is easy to ignore them by joining into &#39;worker&#39; table.</code></pre>
+    You can optionally provide an an external filename or command to get the data from it (don&#39;t forget to append a &#39;|&#39; to the end!)
+    and then the data will be taken from your source and parsed from there.</code></pre>
 
 <h1 id="USAGE-EXAMPLES">USAGE EXAMPLES</h1>
 
-<pre><code>        # Just run it the usual way: query &#39;bacct&#39; and load the relevant data into &#39;lsf_report&#39; table:
-    lsf_report.pl -url mysql://username:secret@hostname:port/long_mult_test
+<pre><code>        # Just run it the usual way: query and store the relevant data into &#39;worker_resource_usage&#39; table:
+    load_resource_usage.pl -url mysql://username:secret@hostname:port/long_mult_test
 
-        # The same, but assuming LSF user someone_else ran the pipeline:
-    lsf_report.pl -url mysql://username:secret@hostname:port/long_mult_test -lsf_user someone_else
+        # The same, but assuming another user &#39;someone_else&#39; ran the pipeline:
+    load_resource_usage.pl -url mysql://username:secret@hostname:port/long_mult_test -username someone_else
 
-        # Assuming the dump file existed. Load the dumped bacct data into &#39;lsf_report&#39; table:
-    lsf_report.pl -url mysql://username:secret@hostname:port/long_mult_test -dump long_mult.bacct
+        # Assuming the dump file existed. Load the dumped bacct data into &#39;worker_resource_usage&#39; table:
+    load_resource_usage.pl -url mysql://username:secret@hostname:port/long_mult_test -source long_mult.bacct
 
-        # Assuming the dump file did not exist. Query &#39;bacct&#39;, dump the data into a file and load it into &#39;lsf_report&#39;:
-    lsf_report.pl -url mysql://username:secret@hostname:port/long_mult_test -dump long_mult_again.bacct</code></pre>
+        # Provide your own command to fetch and parse the worker_resource_usage data from:
+    load_resource_usage.pl -url mysql://username:secret@hostname:port/long_mult_test -source &quot;bacct -l -C 2012/01/25/13:33,2012/01/25/14:44 |&quot;</code></pre>
 
 <h1 id="OPTIONS">OPTIONS</h1>
 
 <pre><code>    -help                   : print this help
     -url &lt;url string&gt;       : url defining where hive database is located
-    -dump &lt;filename&gt;        : a filename for bacct dump. It will be read from if the file exists, and written to otherwise.
-    -lsf_user &lt;username&gt;    : if it wasn&#39;t you who ran the pipeline, LSF user name of that user can be provided
-    -start_date &lt;date&gt;      : minimal start date of a job (the format is &#39;2012/01/25/13:46&#39;)
-    -end_date &lt;date&gt;        : maximal end date of a job (the format is &#39;2012/01/25/13:46&#39;)</code></pre>
+    -username &lt;username&gt;    : if it wasn&#39;t you who ran the pipeline, the name of that user can be provided
+    -source &lt;filename&gt;      : alternative source of worker_resource_usage data. Can be a filename or a pipe-from command.</code></pre>
 
 <h1 id="LICENSE">LICENSE</h1>