code_doc.py 6.68 KB
Newer Older
1

2
# Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
nwillhoft's avatar
nwillhoft committed
3
# Copyright [2016-2021] EMBL-European Bioinformatics Institute
4 5 6 7 8 9 10 11 12 13 14 15 16
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#      http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#####################################################
## Sphinx extension to generate code documentation ##
#####################################################

# The extension provides various directives to embed
# dynamically-generated RestructuredText content from
# the code-base itself
#
# * Schema documentation. This runs sql2rst on the given
#   schema definition files
#
#   .. schema_documentation:: $EHIVE_ROOT_DIR/sql/tables.mysql
#      :foreign_keys: $EHIVE_ROOT_DIR/sql/foreign_keys.sql
#      :title: Hive
#      :embed_diagrams:
#


35
import errno
36
import json
37
import os
38
import os.path
39
import pickle
40 41 42 43 44 45
import subprocess
import sys

from docutils import io, nodes, statemachine
from docutils.parsers.rst import Directive, directives

46 47 48 49 50 51 52 53

# Shamelessly stolen from six
if (sys.version_info[0] == 3):
    string_type = str
else:
    string_type = basestring


54 55 56 57 58 59 60 61 62 63 64
class IncludeCommand(Directive):
    required_arguments = 0
    option_spec = {
            'command': directives.unchanged,
            }

    def run(self):
        content = self.get_content()
        try:
            docutils_input = io.StringInput(source=content)
            rawtext = docutils_input.read()
65
        except IOError as error:
66 67 68 69 70 71 72 73 74 75 76
            # Show the content
            raise self.severe(u'Problems with "%s" command:\n%s.' % ''.join(self.options['command']), ErrorString(error))
        include_lines = statemachine.string2lines(rawtext, 4, convert_whitespace=True)
        self.state_machine.insert_input(include_lines, 'CMD')
        return []

    def get_command(self):
        return self.options['command']

    def get_content(self):
        command = self.get_command()
77
        if isinstance(command, string_type):
78 79 80 81 82 83 84 85 86 87 88 89 90
            return subprocess.check_output(command, stderr=sys.stderr, shell=True)
        else:
            return subprocess.check_output(command, stderr=sys.stderr)


class SchemaDocumentation(IncludeCommand):

    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec = {
            'foreign_keys' : directives.unchanged,
            'title' : directives.unchanged,
91 92
            'sort_headers' : directives.unchanged,
            'sort_tables' : directives.unchanged,
93
            'intro' : directives.unchanged,
Matthieu Muffato's avatar
Matthieu Muffato committed
94
            'url' : directives.unchanged,
95
            'embed_diagrams' : directives.flag,
96
            'cached' : directives.flag,
97 98
            }

99 100 101
    # Where to keep the cached outputs
    cache_filename = os.path.join("_build", "rtd_cache.pickle")

102 103 104 105 106 107 108 109 110 111
    def get_command(self):
        command = [
                os.path.join(os.environ["EHIVE_ROOT_DIR"], "scripts", "dev", "sql2rst.pl"),
                '-i', self.arguments[0].replace('$EHIVE_ROOT_DIR', os.environ["EHIVE_ROOT_DIR"]),
                ]
        self.state.document.settings.record_dependencies.add(command[0], command[2])
        if 'foreign_keys' in self.options:
            foreign_keys_path = self.options['foreign_keys'].replace('$EHIVE_ROOT_DIR', os.environ["EHIVE_ROOT_DIR"])
            self.state.document.settings.record_dependencies.add(foreign_keys_path)
            command.extend( ['--fk', foreign_keys_path] )
112
        for flag in ['embed_diagrams']:
113 114
            if flag in self.options:
                command.extend( ['--' + flag] )
Matthieu Muffato's avatar
Matthieu Muffato committed
115
        for param in ['sort_headers', 'sort_tables', 'url']:
116 117
            if param in self.options:
                command.extend( ['--' + param, self.options[param]] )
118 119 120 121
        if 'intro' in self.options:
            command.extend( ['--intro', self.options['intro'].replace('$EHIVE_ROOT_DIR', os.environ["EHIVE_ROOT_DIR"])] )
        return command

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
    def get_key(self):
        schema_file = self.arguments[0].replace('$EHIVE_ROOT_DIR', os.environ["EHIVE_ROOT_DIR"])
        with open(schema_file, "r") as fh:
            schema = fh.read()
        return (schema, tuple(sorted(self.options.items())))

    def get_cache(self):
        if os.path.exists(self.cache_filename):
            with open(self.cache_filename, "rb") as fh:
                return pickle.load(fh)
        return {}

    def write_cache(self, content_cache):
        with open(self.cache_filename, "wb") as fh:
            pickle.dump(content_cache, fh)

    def get_content(self):
        if 'cached' not in self.options:
            return super(SchemaDocumentation, self).get_content()
        key = self.get_key()
        content_cache = self.get_cache()
        if key in content_cache:
            return content_cache[key]
        content = super(SchemaDocumentation, self).get_content()
        content_cache[key] = content
        self.write_cache(content_cache)
        return content

150 151 152 153 154 155 156 157 158 159 160 161 162 163
class ScriptDocumentation(IncludeCommand):
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec = {}

    def get_command(self):
        script_name = self.arguments[0]
        script_path = os.path.join(os.environ["EHIVE_ROOT_DIR"], "scripts", script_name+".pl")
        self.state.document.settings.record_dependencies.add(script_path)
        # If the command becomes too tricky, we can still decide to implement get_content() instead
        command = '''awk 'BEGIN{p=1} $0 ~ /^=head/ {if (($2 == "NAME") || ($2 == "LICENSE") || ($2 == "CONTACT")) {p=0} else {p=1}} p {print}' %s | pod2html --noindex --title=%s | pandoc --standalone --base-header-level=2 -f html -t rst | sed '/^--/ s/\\\//g' ''' % (script_path, script_name)
        return command

164

165
def cleanup_pod2html_tmp(app, exception):
166 167 168 169 170 171
    # Stolen from https://stackoverflow.com/questions/10840533/most-pythonic-way-to-delete-a-file-which-may-not-exist
    try:
        os.remove("pod2htmd.tmp")
    except OSError as e: # this would be "except OSError, e:" before Python 2.6
        if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
            raise # re-raise exception if a different error occurred
172 173


174 175
def setup(app):
    app.add_directive('schema_documentation', SchemaDocumentation)
176
    app.add_directive('script_documentation', ScriptDocumentation)
177
    app.connect('build-finished', cleanup_pod2html_tmp)
178