Refactor prototype to something close to an MVP (#1)

- translate js to typescript
- add the sequelize ORM for more convenience when interacting with the sqlite database
- enable hierarchical folder structure for storing documents
- remove eleventy

.eleventy.js

-const util = require('util');
-
-module.exports = function(config) {
-  config.addLayoutAlias('default', 'layouts/default.njk');
-  config.addLayoutAlias('with_video', 'layouts/with_video.njk');
-
-  config.addPassthroughCopy('eleventy-site/css');
-  config.addPassthroughCopy('images');
-
-  return {
-    dir: {
-      input: "./eleventy-site",
-      output: "./_site",
-      templateFormats : ["njk", "11ty.js"],
-    }
-  };
-};
-
-
-/*
-
-  // watch for changes in css files (to re-run the postcss pipeline)
-  config.addWatchTarget("css/*.css");
-
-  // Add support for yaml files as data source
-  config.addDataExtension("yaml", contents => yaml.safeLoad(contents));
-
-  // Filter to debug data passed to templates.
-  // Use {{ obj | dump }} to inspect stringified content of the object dumped on the page
-  config.addFilter('dump', obj => {
-    return util.inspect(obj);
-  });
-
-
-  // Add aliases (aliases are in relation to the _includes folder)
-  // Aliases to layouts
-  config.addLayoutAlias('projects', 'layouts/projects.njk');
-  config.addLayoutAlias('default', 'layouts/default.njk');
-
-  // Add contents of the _projects folder to the projects collection
-  config.addCollection("projects", (collection) => collection.getFilteredByGlob("_projects/*.html"));
-
-  // Copy files and folders unchanged to output folder
-  config.addPassthroughCopy("img");
-  config.addPassthroughCopy("CNAME");
-  config.addPassthroughCopy(".nojekyll");
-
-  return {
-    dir: {
-      output: "./_site",
-      templateFormats : ["njk", "md", "11ty.js"],
-    }
-  };
-*/

.gitlab-ci.yml

@@ -30,12 +30,12 @@ variables:
 
 build:
   extends: .build
-  only:
-    - master
+  # only:
+  #   - master
     
 deploy:
   extends: .deploy
   environment:
-    name : ehk-sbox
-  only:
-    - master
+    name : wp-hx-dev
+  # only:
+  #   - master

Dockerfile

 FROM node:12.16.2
 
 ARG SOURCE_DIR="."
+ARG TARGET_DIR="/srv/ensembl-docs-server"
 
-RUN mkdir -p /srv/ensembl-docs-server
+RUN mkdir -p ${TARGET_DIR}
 
-COPY ${SOURCE_DIR} /srv/ensembl-docs-server
+COPY ${SOURCE_DIR} ${TARGET_DIR}
 
-WORKDIR /srv/ensembl-docs-server
+WORKDIR ${TARGET_DIR}
 
-ENV NODE_ENV production
+# FIXME: environment variable should be passed from outside (as build-arg? or at runtime?)
+# Note: the following environment variables have to be defined before the build
+ENV HOST=http://193.62.55.158:30799
 
-RUN npm ci
-RUN npm run build
+RUN npm ci && \
+    npm run build
 
 EXPOSE 3000
 

api/article.js

-const sqlite = require('sqlite');
-const sqlite3 = require('sqlite3');
-
-const config = require('../config');
-
-const getArticleFromDB = require('../scripts/get-from-database/getArticle');
-const databasePath = process.env.DEPLOYMENT === 'NOW' ? 'build/database.db' : config.databasePath;
-
-const getArticle = async (req, res) => {
-  const filePath = req.query.file;
-  if (!filePath) {
-    res.status(400);
-    return res.json({
-      error: 'File parameter cannot be empty'
-    });
-  }
-
-  try {
-    const db = await sqlite.open({
-      filename: databasePath,
-      driver: sqlite3.Database
-    });
-    const match = await getArticleFromDB(db, filePath);
-
-    if (match) {
-      res.setHeader('Access-Control-Allow-Origin', '*');
-      res.json({
-        body: match.body,
-        data: match.data,
-        query: req.query
-      });
-    } else {
-      res.status(404);
-      res.json({
-        error: `File ${filePath} not found`
-      });
-    }
-    await db.close();
-  } catch (error) {
-    console.error('Error opening the database', error);
-    console.error('Database path:', databasePath);
-    res.status(500);
-    res.json({
-      error: 'There was an error processing your request'
-    })
-  }
-};
-
-module.exports = getArticle;

api/diagnostics.js

-const config = require('../config');
-
-const runDiagnostics = async (req, res) => {
-  const databasePath = process.env.DEPLOYMENT === 'NOW' ? 'build/database.db' : config.databasePath;
-  const diagnosticsData = {
-    deploymentEnv: process.env.DEPLOYMENT,
-    databasePath
-  };
-
-  res.json(diagnosticsData);
-};
-
-module.exports = runDiagnostics;

api/search.js

-// const fs = require('fs');
-const sqlite = require('sqlite');
-const sqlite3 = require('sqlite3');
-
-const searchIndex = require('../scripts/searchIndex');
-const getArticleFromDB = require('../scripts/get-from-database/getArticle');
-
-const config = require('../config');
-
-const articlesIndexPath = process.env.DEPLOYMENT === 'NOW'
-  ? 'build/indices/articlesIndex.json'
-  : config.articlesIndexPath;
-const articlesIndex = require(articlesIndexPath);
-
-const databasePath = process.env.DEPLOYMENT === 'NOW'
-  ? 'build/database.db'
-  : config.databasePath;
-
-module.exports = async (req, res) => {
-  let index;
-
-  const { query } = req.query;
-  if (!query) {
-    res.status(400);
-    return res.json({
-      error: 'Query parameter cannot be empty'
-    });
-  }
-
-  const searchResults = searchIndex(query, articlesIndex);
-
-  try {
-    const db = await sqlite.open({
-      filename: databasePath,
-      driver: sqlite3.Database
-    });
-    const slugs = searchResults.map(result => result.ref);
-    const resultPromises = slugs.map(async (slug) => await getArticleFromDB(db, slug));
-    const results = await Promise.all(resultPromises);
-
-    res.setHeader('Access-Control-Allow-Origin', '*');
-    res.json({
-      data: results,
-      query: req.query,
-      // body: req.body,
-      // cookies: req.cookies
-    })
-  } catch (error) {
-    res.status(404);
-    res.json({
-      error,
-      searchResults
-    })
-  }
-}

build.js

-const buildIndex = require('./scripts/build-index');
-const buildDatabase = require('./scripts/build-database');
-
-(async () => {
-  await buildDatabase();
-  await buildIndex();
-})();

config.js → config.ts

-const path = require('path');
+import path from 'path';
+
+const host = process.env.HOST || 'http://127.0.0.1:3000';
+const isProduction = process.env.NODE_ENV === 'production';
+const isBuildingDocs = process.env.NODE_ENV === 'build-docs';
 
 const docsPath = path.resolve(__dirname, 'docs');
-const databaseDirectory = path.resolve(__dirname, 'build');
-const databaseName = 'database.db';
-const indexDirectory = path.resolve(__dirname, 'build/indices');
+const buildPath = isProduction
+  ? path.resolve(__dirname)
+  : path.resolve(__dirname, 'build');
+const databaseDirectory = buildPath; // FIXME?
+const databaseName = 'database.sqlite';
+const indexDirectory = path.resolve(buildPath, 'indices');
 const articlesIndexName = 'articlesIndex.json';
+const buildImagesPath = path.resolve(buildPath, 'images');
 
-module.exports = {
+export default {
+  host,
+  isProduction,
+  isBuildingDocs,
+  buildPath,
+  buildImagesPath,
   docsPath,
   articlesPath: path.join(docsPath, 'article'),
   videosPath: path.join(docsPath, 'video'),

decisions/entities.md

+# Article
+## Fields
+- slug — required
+- title - required
+- tags — optional
+- status — optional?
+- parent - optional
+- next - optional
+- body - required
+
+(when a database is built, they will also have the field "path")
+
+# Video
+## Fields
+- slug — required
+- type: 'video'
+- title - required
+- description - optional
+- tags - optional
+- next - optional
+
+
+
+Validation

decisions/questions.md

+- should slug be globally unique or only unique within a folder?
+- how shall the help&docs app create pages?
+  - what is the entry point?
+  - what should the api response look like?
+- what does a section page (directory page) look like? Is there a common structure for such pages, or are they all different?
+
+Example of a huge TOC file in Azure docs: `https://github.com/MicrosoftDocs/azure-docs/blob/master/articles/azure-sql/toc.yml`

decisions/todo.md

+# TODO
+- Fetch all articles for a collection (paginated?)
+- Link related articles
+- Figure out what to do with index pages
+
+## Basic server-side maintenance
+- Add healthcheck endpoint
+- Add proper logging
+
+
+
+
+# Backend
+- Enable api to report menu structure
+  - Q: how many apps (i.e. what data) is the app going to support?
+    - contextual help
+    - Help&Docs app
+    - About Ensembl app?
+    - Ensembl static data (species pages)?
+- Enable entrypoint through the api to build pages
+  - should the entrypoint be just the menu, and then individual pages are requested through subsequent requests, or should all pages be returned en masse (alternatively, as a paginated list)?
+- Rebuild data generation (recurse through the folder structure)
+  - decide what to use for page identifier
+  - make sure images are copied to relevant folder and can be served correctly
+- Decide how to handle section pages (example: getting started page, using Ensembl page, Known bugs page, Contact page). Should we support html pages as well
+
+# Client
+- be able to show pages
+- be able to navigate between pages
+- notice that text and video sections in contextual menu may change independently when you click on a related article/video link
+  - Q: Are contextual menu and page in help&docs app same pages or different?
+
+
+# Consequences
+## Content ids
+- using file paths for ids
+  - Advantages
+    - ensures uniqueness
+    - nicely reflect the url
+    - do not require any involvement from content creators
+  - Disadvantages
+    - things may break if files are moved or renamed
+    - a bit cumbersome in api requests (need to uri-escape slashes)
+- using slugs (or other kind of id)
+  - Advantages
+    - resilience to changes in file system
+  - Disadvantages
+    - requires manual input from content creators
+    - need extra validation code to guarantee uniqueness
+
+It is potentially possible to have both (with slug/id being a potentially optional field)
+
+## Related articles
+If relations are established through the same tag/category, then the order of the articles will be determined automatically (we could sort alphabetically, of course). I suspect that content creators will want full control over the order of related articles, in which case relations should be written out manually.

docs/article/ensembl-select.md

----
-parent: viewing-ensembl-data
-tags:
-  - browser
-  - transcript
-status: published
----
-
-# What is Ensembl Select?
-
-<style>
-.scary > blockquote {
-  background-color: rgba(237, 51, 21, 0.2);
-  border-left-color: #ed3315;
-}
-</style>
-
-<div class="scary">
-
->Caution:
->
->This page describes **experimental features that are not yet available in a stable release**. Don't rely on experimental builds of React in production apps. These features may change significantly and without a warning before they become a part of React.
->
->This documentation is aimed at early adopters and people who are curious. **If you're new to React, don't worry about these features** -- you don't need to learn them right now.
-
-</div>
-
-<iframe width="560" height="315" src="https://www.youtube.com/embed/C2g37X_uMok" frameborder="0" allowfullscreen></iframe>
-
-
-# What is Ensembl Select?
-
-Cras id arcu porttitor, luctus turpis eu, venenatis risus. Curabitur iaculis mauris vitae ipsum euismod, in mattis nisi suscipit. Vestibulum tincidunt suscipit eros. Integer gravida dolor in mi tincidunt, ut luctus lectus consectetur. In vestibulum blandit ipsum, sed pulvinar ipsum posuere eget. 

docs/article/getting-started.md

----
-status: published
----
-
-# Getting Started
-
-Nullam dapibus ligula id dolor molestie, vel lacinia ante consequat. Vestibulum eu magna neque. Integer fermentum ante sit amet ante sagittis, ut mollis lacus maximus. Maecenas cursus magna molestie volutpat lacinia. Fusce vestibulum, elit sit amet semper mattis, quam lectus accumsan felis, eget luctus massa tortor et mauris. Maecenas faucibus dui ex.

docs/article/viewing-ensembl-data.md

----
-parent: using-ensembl
-status: published
----
-
-# Viewing Ensembl data
-
-Curabitur in auctor urna, ac sollicitudin massa. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam pellentesque sem ac dictum finibus. Suspendisse iaculis scelerisque quam quis bibendum. Nullam augue felis, laoreet at laoreet vel, mattis sit amet risus. Aliquam imperdiet, enim ut feugiat egestas, augue odio mollis sapien, eu suscipit tellus ante et ligula.

docs/ensembl-help/getting-started/about-the-site.md

+---
+status: published
+---
+
+# About the site

docs/ensembl-help/getting-started/faq.md

+---
+status: published
+---
+
+# Frequently asked questions

docs/ensembl-help/getting-started/index.yml

+metadata:
+  title: Getting Started
+  description: Learn how to start using Ensembl website
+  type: landing-page
+
+content:
+
+  - title: What is Ensembl
+    description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam egestas mauris condimentum tempor vulputate.
+    link_text: See more
+    link: what-is-ensembl.md
+
+  - title: About the site
+    description: Sed luctus cursus urna, eu ornare mi ullamcorper gravida. Sed et congue purus.
+    link_text: See more
+    link: about-the-site.md

docs/ensembl-help/getting-started/search.md

+---
+title: Search
+description: How to search Ensembl website
+status: published
+---
+
+# Search

docs/article/select-a-species.md → docs/ensembl-help/getting-started/selecting-a-species.md

 ---
-parent: getting-started
-related-video: select-a-species
+slug: selecting-a-species
+title: Select a species or assembly
+description: In order to start using Ensembl website, you first need to select a species.
+related_videos:
+  - relativePath: videos/select-a-species.yml
+related_articles:
+  - relativePath: search.md
+  - docsRootPath: ensembl-help/using-ensembl/viewing-ensembl-data/genome-browser/using-the-genome-browser.md
 status: published
 ---
 
 # Select a species or assembly
 
-Here is example image: ![image](/images/foo/american-red-squirrel.jpg)
+## This is updated text 
+
+Here is example image: ![image](media/american-red-squirrel.jpg)
 
 Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam egestas mauris condimentum tempor vulputate. Sed luctus cursus urna, eu ornare mi ullamcorper gravida. Sed et congue purus. Pellentesque eu turpis sit amet ligula malesuada condimentum quis vitae justo. Aenean mattis efficitur nisl. Aenean tortor leo, tempor a justo eget, tincidunt faucibus purus. Praesent at lacinia ligula. Proin lacus justo, finibus vehicula condimentum a, auctor eget tortor. Nam feugiat imperdiet lectus, sed pellentesque arcu facilisis et. Duis vel est lorem. Mauris varius lectus eget tincidunt lacinia.