Commit b48d23cc authored by Leo Gordon's avatar Leo Gordon
Browse files

updated schema diagram (PNG) and description (HTML)

parent e747d057
<html>
<head>
<meta http-equiv="CONTENT-TYPE" content="text/html; charset=utf-8" />
<title>Hive Schema Documentation</title>
<link rel="stylesheet" type="text/css" media="all" href="http://static.ensembl.org/minified/eb8658698ad4d45258b954c2d3f35bad.css" />
<meta name="order" content="2" />
<link rel="stylesheet" type="text/css" media="all" href="http://static.ensembl.org/minified/9256d71d74fd7c3e9e2b7815edb7812b.css" />
<script language="Javascript" type="text/javascript">
var img_plus = '<img src="/i/16/plus-button.png" style="width:12px;height:12px;vertical-align:middle" alt="show"/>';
......@@ -73,8 +72,6 @@
<body>
<h1>Ensembl Hive Schema Documentation</h1>
<h2>Introduction</h2>
<p><i>please, insert your introduction here</i><p><br />
<h3 id="top">List of the tables:</h3>
<div>
......@@ -155,7 +152,7 @@
<br /><br />
<div style="background-color:#F2F2F2;padding:5px;margin:5px 0px;border-top:2px solid #C70C09">
<h2 style="display:inline;color:#000">Pipeline structure</h2>
<h2 id="1" style="display:inline;color:#000">Pipeline structure</h2>
</div>
<div id="analysis_base" style="width:820px;height:20px;border: 2px groove #CCC;background-color:#FAFAFF;padding:2px;margin-top:35px;margin-bottom:2px">
......@@ -175,7 +172,7 @@
<table style="border:1px solid #667aa6;padding:0px;min-width:1000px;max-width:1200px">
<tr class="center" style="color:#FFFFFF;background-color:#667aa6"><th style="color:#FFF;padding:2px">Column</th><th style="color:#FFF;padding:2px">Type</th><th style="color:#FFF;padding:2px;min-width:80px">Default value</th><th style="color:#FFF;padding:2px;min-width:500px">Description</th><th style="color:#FFF;padding:2px;min-width:100px">Index</th></tr>
<tr class="bg1"><td><b>analysis_id</b></td><td>INTEGER</td><td></td><td>a unique ID that is also a foreign key to most of the other tables</td><td></td></tr>
<tr class="bg2"><td><b>logic_name</b></td><td>VARCHAR(40)</td><td></td><td>the name of the Analysis object</td><td>unique key: logic_name_idx</td></tr>
<tr class="bg2"><td><b>logic_name</b></td><td>VARCHAR(255)</td><td></td><td>the name of the Analysis object</td><td>unique key: logic_name_idx</td></tr>
<tr class="bg1"><td><b>module</b></td><td>VARCHAR(255)</td><td></td><td>the Perl module name that runs this Analysis</td><td></td></tr>
<tr class="bg2"><td><b>parameters</b></td><td>TEXT</td><td></td><td>a stingified hash of parameters common to all jobs of the Analysis</td><td></td></tr>
<tr class="bg1"><td><b>resource_class_id</b></td><td>INTEGER</td><td></td><td>link to the resource_class table</td><td></td></tr>
......@@ -183,7 +180,7 @@
<tr class="bg1"><td><b>max_retry_count</b></td><td>INTEGER</td><td>3</td><td>how many times a job of this Analysis will be retried (unless there is no point)</td><td></td></tr>
<tr class="bg2"><td><b>can_be_empty</b></td><td>SMALLINT</td><td>0</td><td>if TRUE, this Analysis will not be blocking if/while it doesn't have any jobs</td><td></td></tr>
<tr class="bg1"><td><b>priority</b></td><td>SMALLINT</td><td>0</td><td>an Analysis with higher priority will be more likely chosen on Worker's specialization</td><td></td></tr>
<tr class="bg2"><td><b>meadow_type</b></td><td>VARCHAR(40)</td><td>NULL</td><td>if defined, forces this Analysis to be run only on the given Meadow</td><td></td></tr>
<tr class="bg2"><td><b>meadow_type</b></td><td>VARCHAR(255)</td><td>NULL</td><td>if defined, forces this Analysis to be run only on the given Meadow</td><td></td></tr>
<tr class="bg1"><td><b>analysis_capacity</b></td><td>INTEGER</td><td>NULL</td><td>if defined, limits the number of Workers of this particular Analysis that are allowed to run in parallel</td><td></td></tr>
</table>
</div>
......@@ -275,7 +272,7 @@
<br /><br />
<div style="background-color:#F2F2F2;padding:5px;margin:5px 0px;border-top:2px solid #FF7504">
<h2 style="display:inline;color:#000">Resources</h2>
<h2 id="2" style="display:inline;color:#000">Resources</h2>
</div>
<div id="resource_class" style="width:820px;height:20px;border: 2px groove #CCC;background-color:#FAFAFF;padding:2px;margin-top:35px;margin-bottom:2px">
......@@ -295,7 +292,7 @@
<table style="border:1px solid #667aa6;padding:0px;min-width:1000px;max-width:1200px">
<tr class="center" style="color:#FFFFFF;background-color:#667aa6"><th style="color:#FFF;padding:2px">Column</th><th style="color:#FFF;padding:2px">Type</th><th style="color:#FFF;padding:2px;min-width:80px">Default value</th><th style="color:#FFF;padding:2px;min-width:500px">Description</th><th style="color:#FFF;padding:2px;min-width:100px">Index</th></tr>
<tr class="bg1"><td><b>resource_class_id</b></td><td>INTEGER</td><td></td><td>unique ID of the ResourceClass</td><td></td></tr>
<tr class="bg2"><td><b>name</b></td><td>VARCHAR(40)</td><td></td><td>unique name of the ResourceClass</td><td>unique: key</td></tr>
<tr class="bg2"><td><b>name</b></td><td>VARCHAR(255)</td><td></td><td>unique name of the ResourceClass</td><td>unique: key</td></tr>
</table>
</div>
......@@ -316,14 +313,15 @@
<table style="border:1px solid #667aa6;padding:0px;min-width:1000px;max-width:1200px">
<tr class="center" style="color:#FFFFFF;background-color:#667aa6"><th style="color:#FFF;padding:2px">Column</th><th style="color:#FFF;padding:2px">Type</th><th style="color:#FFF;padding:2px;min-width:80px">Default value</th><th style="color:#FFF;padding:2px;min-width:500px">Description</th><th style="color:#FFF;padding:2px;min-width:100px">Index</th></tr>
<tr class="bg1"><td><b>resource_class_id</b></td><td>INTEGER</td><td></td><td>foreign-keyed to the ResourceClass entry</td><td>primary key</td></tr>
<tr class="bg2"><td><b>meadow_type</b></td><td>VARCHAR(40)</td><td></td><td>if the Worker is about to be executed on the given Meadow...</td><td>primary key</td></tr>
<tr class="bg1"><td><b>parameters</b></td><td>VARCHAR(255)</td><td>''</td><td>... the following resource line should be given to it.</td><td></td></tr>
<tr class="bg2"><td><b>meadow_type</b></td><td>VARCHAR(255)</td><td></td><td>if the Worker is about to be executed on the given Meadow...</td><td>primary key</td></tr>
<tr class="bg1"><td><b>submission_cmd_args</b></td><td>VARCHAR(255)</td><td>''</td><td>... these are the resource arguments (queue, memory,...) to give to the submission command</td><td></td></tr>
<tr class="bg2"><td><b>worker_cmd_args</b></td><td>VARCHAR(255)</td><td>''</td><td>... and these are the arguments that are given to the worker command being submitted</td><td></td></tr>
</table>
</div>
<br /><br />
<div style="background-color:#F2F2F2;padding:5px;margin:5px 0px;border-top:2px solid #1D73DA">
<h2 style="display:inline;color:#000">Job-related</h2>
<h2 id="3" style="display:inline;color:#000">Job-related</h2>
</div>
<div id="job" style="width:820px;height:20px;border: 2px groove #CCC;background-color:#FAFAFF;padding:2px;margin-top:35px;margin-bottom:2px">
......@@ -444,7 +442,7 @@
<div id="div_hive_meta" style="display:none">
<table style="border:1px solid #667aa6;padding:0px;min-width:1000px;max-width:1200px">
<tr class="center" style="color:#FFFFFF;background-color:#667aa6"><th style="color:#FFF;padding:2px">Column</th><th style="color:#FFF;padding:2px">Type</th><th style="color:#FFF;padding:2px;min-width:80px">Default value</th><th style="color:#FFF;padding:2px;min-width:500px">Description</th><th style="color:#FFF;padding:2px;min-width:100px">Index</th></tr>
<tr class="bg1"><td><b>meta_key</b></td><td>VARCHAR(80)</td><td></td><td>the KEY of KEY-VALUE pairs (primary key)</td><td></td></tr>
<tr class="bg1"><td><b>meta_key</b></td><td>VARCHAR(255)</td><td></td><td>the KEY of KEY-VALUE pairs (primary key)</td><td></td></tr>
<tr class="bg2"><td><b>meta_value</b></td><td>TEXT</td><td></td><td>the VALUE of KEY-VALUE pairs</td><td></td></tr>
</table>
</div>
......@@ -467,14 +465,14 @@
<tr class="center" style="color:#FFFFFF;background-color:#667aa6"><th style="color:#FFF;padding:2px">Column</th><th style="color:#FFF;padding:2px">Type</th><th style="color:#FFF;padding:2px;min-width:80px">Default value</th><th style="color:#FFF;padding:2px;min-width:500px">Description</th><th style="color:#FFF;padding:2px;min-width:100px">Index</th></tr>
<tr class="bg1"><td><b>meta_id</b></td><td>INTEGER</td><td></td><td>auto-incrementing primary key, not really used per se</td><td></td></tr>
<tr class="bg2"><td><b>species_id</b></td><td>INTEGER</td><td>1</td><td>always 1, kept for compatibility with the Core API</td><td>unique key: species_key_value_idx<br />key: species_value_idx</td></tr>
<tr class="bg1"><td><b>meta_key</b></td><td>VARCHAR(40)</td><td></td><td>the KEY of KEY-VALUE pairs</td><td>unique key: species_key_value_idx</td></tr>
<tr class="bg1"><td><b>meta_key</b></td><td>VARCHAR(255)</td><td></td><td>the KEY of KEY-VALUE pairs</td><td>unique key: species_key_value_idx</td></tr>
<tr class="bg2"><td><b>meta_value</b></td><td>TEXT</td><td></td><td>the VALUE of KEY-VALUE pairs</td><td>unique key: species_key_value_idx<br />key: species_value_idx</td></tr>
</table>
</div>
<br /><br />
<div style="background-color:#F2F2F2;padding:5px;margin:5px 0px;border-top:2px solid #24DA06">
<h2 style="display:inline;color:#000">worker table</h2>
<h2 id="4" style="display:inline;color:#000">worker table</h2>
</div>
<div id="worker" style="width:820px;height:20px;border: 2px groove #CCC;background-color:#FAFAFF;padding:2px;margin-top:35px;margin-bottom:2px">
......@@ -494,10 +492,10 @@
<table style="border:1px solid #667aa6;padding:0px;min-width:1000px;max-width:1200px">
<tr class="center" style="color:#FFFFFF;background-color:#667aa6"><th style="color:#FFF;padding:2px">Column</th><th style="color:#FFF;padding:2px">Type</th><th style="color:#FFF;padding:2px;min-width:80px">Default value</th><th style="color:#FFF;padding:2px;min-width:500px">Description</th><th style="color:#FFF;padding:2px;min-width:100px">Index</th></tr>
<tr class="bg1"><td><b>worker_id</b></td><td>INTEGER</td><td></td><td>unique ID of the Worker</td><td></td></tr>
<tr class="bg2"><td><b>meadow_type</b></td><td>VARCHAR(40)</td><td></td><td>type of the Meadow it is running on</td><td></td></tr>
<tr class="bg1"><td><b>meadow_name</b></td><td>VARCHAR(40)</td><td></td><td>name of the Meadow it is running on (for 'LOCAL' type is the same as host)</td><td></td></tr>
<tr class="bg2"><td><b>host</b></td><td>VARCHAR(40)</td><td></td><td>execution host name</td><td></td></tr>
<tr class="bg1"><td><b>process_id</b></td><td>VARCHAR(40)</td><td></td><td>identifies the Worker process on the Meadow (for 'LOCAL' is the OS PID)</td><td></td></tr>
<tr class="bg2"><td><b>meadow_type</b></td><td>VARCHAR(255)</td><td></td><td>type of the Meadow it is running on</td><td></td></tr>
<tr class="bg1"><td><b>meadow_name</b></td><td>VARCHAR(255)</td><td></td><td>name of the Meadow it is running on (for 'LOCAL' type is the same as host)</td><td></td></tr>
<tr class="bg2"><td><b>host</b></td><td>VARCHAR(255)</td><td></td><td>execution host name</td><td></td></tr>
<tr class="bg1"><td><b>process_id</b></td><td>VARCHAR(255)</td><td></td><td>identifies the Worker process on the Meadow (for 'LOCAL' is the OS PID)</td><td></td></tr>
<tr class="bg2"><td><b>resource_class_id</b></td><td>INTEGER</td><td>NULL</td><td>links to Worker's resource class</td><td></td></tr>
<tr class="bg1"><td><b>analysis_id</b></td><td>INTEGER</td><td>NULL</td><td>Analysis the Worker is specified into</td><td>key: analysis_status</td></tr>
<tr class="bg2"><td><b>work_done</b></td><td>INTEGER</td><td>0</td><td>how many jobs the Worker has completed successfully</td><td></td></tr>
......@@ -505,14 +503,14 @@
<tr class="bg2"><td><b>born</b></td><td>TIMESTAMP</td><td>CURRENT_TIMESTAMP</td><td>when the Worker process was started</td><td></td></tr>
<tr class="bg1"><td><b>last_check_in</b></td><td>TIMESTAMP</td><td></td><td>when the Worker last checked into the database</td><td></td></tr>
<tr class="bg2"><td><b>died</b></td><td>TIMESTAMP</td><td></td><td>if defined, when the Worker died (or its premature death was first detected by GC)</td><td></td></tr>
<tr class="bg1"><td><b>cause_of_death</b></td><td>ENUM('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN')</td><td>NULL</td><td>if defined, why did the Worker exit (or why it was killed)</td><td></td></tr>
<tr class="bg1"><td><b>cause_of_death</b></td><td>ENUM('NO_ROLE', 'NO_WORK', 'JOB_LIMIT', 'HIVE_OVERLOAD', 'LIFESPAN', 'CONTAMINATED', 'RELOCATED', 'KILLED_BY_USER', 'MEMLIMIT', 'RUNLIMIT', 'SEE_MSG', 'UNKNOWN')</td><td>NULL</td><td>if defined, why did the Worker exit (or why it was killed)</td><td></td></tr>
<tr class="bg2"><td><b>log_dir</b></td><td>VARCHAR(255)</td><td>NULL</td><td>if defined, a filesystem directory where this Worker's output is logged</td><td></td></tr>
</table>
</div>
<br /><br />
<div style="background-color:#F2F2F2;padding:5px;margin:5px 0px;border-top:2px solid #F4D20C">
<h2 style="display:inline;color:#000">Logging and monitoring</h2>
<h2 id="5" style="display:inline;color:#000">Logging and monitoring</h2>
</div>
<div id="log_message" style="width:820px;height:20px;border: 2px groove #CCC;background-color:#FAFAFF;padding:2px;margin-top:35px;margin-bottom:2px">
......
No preview for this file type
docs/hive_schema.png

239 KB | W: | H:

docs/hive_schema.png

236 KB | W: | H:

docs/hive_schema.png
docs/hive_schema.png
docs/hive_schema.png
docs/hive_schema.png
  • 2-up
  • Swipe
  • Onion skin
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment