Update the API to support the Help app (#24)

docs/ensembl-help/getting-started-old/videos/find-and-add-species.yml

+type: video
+title: How to find and add species
+description: How to find and add a species in the Species Selector App
+youtube_id: egV43n8nRYI

docs/ensembl-help/getting-started/about-the-site.md

+---
+title: About the site
+description: Quick tour around the site
+---
+
+This is a placeholder for the _About the site_ article.

docs/ensembl-help/getting-started/index.yml

+type: index
+title: Getting started
+description: Introduction to using the Ensembl web site
+items:
+  - title: What is Ensembl
+    summary: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ut tempus mi, eget congue sapien. Vivamus tempus nec purus eget mollis.
+    href: what-is-ensembl.md
+  - title: About the site
+    summary: Morbi eget orci id ex porta elementum luctus id ipsum. Praesent quis ligula non nisl aliquet blandit. Vivamus dapibus turpis diam, venenatis fringilla eros semper ut.
+    href: about-the-site.md
+  - title: Selecting a species
+    summary: Sed dictum turpis nec condimentum efficitur. Sed eu aliquam lacus. Sed posuere rutrum elit id mattis. Quisque iaculis odio vel ipsum tincidunt bibendum.
+    href: selecting-a-species.md
+  - title: Search
+    summary: Aenean faucibus blandit purus bibendum tincidunt. Nam pharetra aliquam condimentum. Sed sit amet dignissim purus. Fusce vehicula eget sapien et congue.
+    href: search.md

docs/ensembl-help/getting-started/search.md

+---
+title: Getting started — Search
+description: Brief summary of how to search the Ensembl site
+---
+
+This is a placeholder for the _Search_ article.

docs/ensembl-help/getting-started/selecting-a-species.md

+---
+title: Selecting a species
+description: Here is how to select a species
+---
+
+This is a placeholder for the _Selecting a species_ article.

docs/ensembl-help/getting-started/what-is-ensembl.md

+---
+title: What is Ensembl
+description: Brief summary of what Ensembl is
+---
+
+This is a placeholder for the _What is Ensembl_ article.

docs/ensembl-help/toc.yml

 - name: Getting Started
-  href: getting-started/toc.yml
-  url: /getting-started
+  href: getting-started/index.yml
+  url: /help
+- name: Something Other than Getting Started
+  href: getting-started-old/toc.yml
 - name: Using Ensembl
   href: using-ensembl/toc.yml

package-lock.json

@@ -47,7 +47,8 @@
         "ts-node-dev": "1.1.1",
         "typescript": "4.1.5",
         "unified": "9.2.0",
-        "unist-util-visit": "2.0.3",
+        "unist-util-is": "4.0.2",
+        "unist-util-visit-parents": "3.1.0",
         "yaml": "1.10.0"
       }
     },

package.json

@@ -56,7 +56,8 @@
     "ts-node-dev": "1.1.1",
     "typescript": "4.1.5",
     "unified": "9.2.0",
-    "unist-util-visit": "2.0.3",
+    "unist-util-is": "4.0.2",
+    "unist-util-visit-parents": "3.1.0",
     "yaml": "1.10.0"
   }
 }

src/build-scripts/build-database/addArticles.ts

 import path from 'path';
+import pick from 'lodash/pick';
 
 import {
   fromDocumentsRoot
 } from '../filePathHelpers';
 
 import { Article, Collection } from '../../models';
-import { TextArticle } from '../../models/Article';
+import { IndexArticle, TextArticle } from '../../models/Article';
 
+import { ParsedIndexPage } from '../../types/ParsedIndexPage';
 import { ParsedArticle } from '../../types/ParsedArticle';
 import { ParsedVideo } from '../../types/ParsedVideo';
 
-type ParsedFile = ParsedArticle | ParsedVideo;
+type ParsedFile = ParsedIndexPage| ParsedArticle | ParsedVideo;
 
 const addArticles = async (items: ParsedFile[]) => {
   let savedArticles: { parsedFile: ParsedFile, savedArticle: Article }[] = [];
@@ -20,12 +22,18 @@ const addArticles = async (items: ParsedFile[]) => {
       savedArticle: await saveArticle(item)
     });
   }
+  // now that all articles have been saved to the database,
+  // it's time to establish relationships between them
   for (const item of savedArticles) {
-    await addRelationships(item);
+    if (item.parsedFile.type === 'index') {
+      await addLinksToIndexArticles(item.savedArticle as IndexArticle);
+    } else {
+      await addRelationships(item);
+    }
   }
 };
 
-const saveArticle = async (article: ParsedArticle | ParsedVideo) => {
+const saveArticle = async (article: ParsedFile) => {
   const newArticle = Article.create({
     title: article.title || 'empty title',
     type: article.type,
@@ -46,9 +54,16 @@ const saveArticle = async (article: ParsedArticle | ParsedVideo) => {
   return newArticle;
 };
 
-const prepareArticleMetadata = (article: ParsedArticle | ParsedVideo) => {
+const prepareArticleMetadata = (article: ParsedFile) => {
   if (article.type === 'video') {
     return { youtube_id: article.youtube_id };
+  } else if (article.type === 'index') {
+    return {
+      items: article.items.map(item => ({
+        ...pick(item, ['title', 'summary']),
+        url: item.href
+      }))
+    };
   }
 }
 
@@ -72,7 +87,7 @@ const addRelationships = async (item: { parsedFile: ParsedFile, savedArticle: Ar
     return;
   }
 
-  if (parsedFile.related_articles) {
+  if ('related_articles' in parsedFile) {
     for (const { href } of parsedFile.related_articles) {
       const pathToRelatedArticle = buildPathToRelatedFile(savedArticle.filePath, href);
       const relatedArticle = await Article.findOne({ where: { filePath: pathToRelatedArticle } });
@@ -95,6 +110,28 @@ const addRelationships = async (item: { parsedFile: ParsedFile, savedArticle: Ar
   await savedArticle.save();
 };
 
+const addLinksToIndexArticles = async (indexArticle: IndexArticle) => {
+  const filePath = indexArticle.filePath;
+  const indexItems = indexArticle.data.items;
+  for (const indexItem of indexItems) {
+    // At this point, the url field of an index item of the saved article
+    // contains the relative path to the file that the index article is referencing.
+    // Here, we will overwrite this field with the actual url.
+    // TODO: should probably first check that the href is not a url to an external article
+    const pathToLinkedArticle = buildPathToRelatedFile(filePath, indexItem.url);
+    const savedLinkedArticle = await Article.findOne({ where: { filePath: pathToLinkedArticle } });
+
+    if (!savedLinkedArticle) {
+      console.log('Incorrect path for related article provided:', pathToLinkedArticle);
+      continue;
+    }
+
+    indexItem.url = savedLinkedArticle.url;
+  }
+
+  await indexArticle.save();
+};
+
 const buildPathToRelatedFile = (sourceFilePath: string, relatedFilePath: string) => {
   const { dir: sourceFileDirectory } = path.parse(sourceFilePath);
   return path.join(sourceFileDirectory, relatedFilePath);

src/build-scripts/build-database/imagePlugin.ts

+import path from 'path';
+import visitParents from 'unist-util-visit-parents';
+import unistUtilIs from 'unist-util-is';
+import { Node } from 'unist';
+
+import config from '../../../config';
+
+const imagePlugin = () => (tree: Node, file: any) => {
+  // check that the node is an image element
+  const test = (node: Node): node is Node =>
+    unistUtilIs(node, { tagName: 'img' });
+
+  visitParents(
+    tree,
+    test,
+    (node: Node, ancestors) => {
+      updateImagePath(node as Element, file);
+      updateImageWrapperElement(ancestors as Element[]);
+    }
+  );
+};
+
+type Element = Node & {
+  type: 'element';
+  tagName: string;
+  properties: Record<string, unknown>;
+  children: Element[];
+};
+
+const updateImagePath = (imageNode: Element, file: any) => {
+  const imageSource = imageNode.properties.src as string;
+  // FIXME: check image source; only do the following if it's not an absolute url
+
+  const filePath = file.path; // this is the path of the markdown file containing the reference
+  const { dir: directoryPath } = path.parse(filePath);
+
+  // image files will be copied to the destination folder preserving the original directory tree
+  const directoryRelativeToDocsRoot = path.relative(config.docsPath, directoryPath);
+  const destDirectory = '/api/docs/images'; // this is where images are going to be copied
+  const destImagePath = path.join(destDirectory, directoryRelativeToDocsRoot, imageSource);
+
+
+  imageNode.properties.src = destImagePath;
+};
+
+const updateImageWrapperElement = (imageAncestors: Element[]) => {
+  const immediateAncestor = imageAncestors[imageAncestors.length - 1];
+  immediateAncestor.tagName = 'figure';
+};
+
+
+export default imagePlugin;

src/build-scripts/build-database/markdownImagePlugin.ts

-import visit from 'unist-util-visit';
-import { Node } from 'unist';
-
-import config from '../../../config';
-
-const attacher = () => {
-  const transformer = (tree: Node, file: any) => {
-    // FIXME: the file parameter is vfile
-    const { path: filePath } = file;
-    visit(tree, 'image', imageVisitor(filePath));
-  };
-
-  return transformer;
-};
-
-const imageVisitor = (filePath: string) => (node: Node) => {
-  const markdownDirectory = filePath.substring(config.docsPath.length + 1)
-    .split('/')
-    .slice(0, -1)
-    .join('/');
-  const destPath = `/api/docs/images/${markdownDirectory}/${node.url}`;
-
-  node.url = destPath;
-};
-
-export default attacher;

src/build-scripts/build-database/parseMarkdown.ts

@@ -12,7 +12,7 @@ import frontmatter from 'remark-frontmatter';
 import html from 'rehype-stringify';
 import yaml from 'yaml';
 
-import imagePlugin from './markdownImagePlugin';
+import imagePlugin from './imagePlugin';
 
 const parseMarkdown = async (pathToFile: string) => {
   const processedFile = await
@@ -20,8 +20,8 @@ const parseMarkdown = async (pathToFile: string) => {
       .use(parse)
       .use(frontmatter, ['yaml', 'toml'])
       .use(extract, { yaml: yaml.parse })
-      .use(imagePlugin)
       .use(remark2rehype, {allowDangerousHtml: true})
+      .use(imagePlugin)
       .use(raw)
       .use(html)
       .process(vfile.readSync(pathToFile));

src/build-scripts/build-database/readSourceFiles.ts

@@ -152,12 +152,10 @@ const buildUrlForArticleWithoutMenu = (article: { path: string, type: string, ti
   const urlNamespace = pathToMenuNameMap.get(dirName);
 
   if (!urlNamespace) {
-    console.log({ dirName, urlNamespace, pathToMenuNameMap });
-
     throw new Error(`Could not create url for file ${article.path}`);
   }
 
-  return buildPageUrl(article.title, article.type, urlNamespace);
+  return buildPageUrl(article.title, article.type, `/${urlNamespace}`);
 };
 
 const isMarkdownFile = (filePath: string) => path.parse(filePath).ext === '.md';

src/controllers/articlesController.ts

@@ -3,7 +3,7 @@ import { In } from "typeorm";
 import pick from 'lodash/pick';
 
 import { Article } from '../models';
-import { TextArticle, VideoArticle } from '../models/Article';
+import { IndexArticle, TextArticle, VideoArticle } from '../models/Article';
 
 export const getArticle = async (req: Request, res: Response) => {
   const slug = req.query.slug as string | undefined;
@@ -24,16 +24,13 @@ export const getArticle = async (req: Request, res: Response) => {
     }) as TextArticle | VideoArticle | null;
 
     if (article) {
-      const relatedArticleIds = article.data?.relatedArticles || [];
-      const relatedArticles = await Article.find({ id: In(relatedArticleIds) });
       res.json({
         slug: article.slug,
         type: article.type,
         url: article.url,
         title: article.title,
         description: article.description,
-        related_articles: relatedArticles.map(article => pick(article, ['title', 'type', 'url', 'slug'])),
-        ...getTypeSpecificArticleFields(article)
+        ...await getTypeSpecificArticleFields(article)
       });
     } else {
       res.status(404);
@@ -50,13 +47,27 @@ export const getArticle = async (req: Request, res: Response) => {
   }
 };
 
-const getTypeSpecificArticleFields = (article: TextArticle |VideoArticle) => {
+const getTypeSpecificArticleFields = async (article: TextArticle | VideoArticle | IndexArticle) => {
   switch (article.type) {
     case 'article':
-      return { body: article.body };
+      return {
+        body: article.body,
+        related_articles: await populateRelatedArticles(article)
+      };
     case 'video':
-      return { youtube_id: article.data.youtube_id };
+      return {
+        youtube_id: article.data.youtube_id,
+        related_articles: await populateRelatedArticles(article)
+      };
+    case 'index':
+      return { items: article.data.items };
     default:
       return null;
   }
-}
+};
+
+const populateRelatedArticles = async (article: TextArticle | VideoArticle) => {
+  const relatedArticleIds = article.data?.relatedArticles || [];
+  const relatedArticles = await Article.find({ id: In(relatedArticleIds) });
+  return relatedArticles.map(article => pick(article, ['title', 'type', 'url', 'slug']));
+};

src/models/Article.ts

@@ -8,6 +8,8 @@ import {
 
 import { Collection } from './Collection';
 
+import { IndexPageItem } from '../types/ParsedIndexPage';
+
 @Entity()
 export class Article extends BaseEntity {
 
@@ -63,3 +65,17 @@ export type VideoArticle = Article & {
     relatedArticles?: number[];
   };
 };
+
+export type IndexArticle = Article & {
+
+  type: 'index';
+
+  data: {
+    items: Array<{
+      title: string;
+      summary: string;
+      url: string;
+    }>
+  };
+
+};