Skip to content
Snippets Groups Projects
Unverified Commit dc85a0bc authored by Andrey Azov's avatar Andrey Azov Committed by GitHub
Browse files

Refactor prototype to something close to an MVP (#1) 

- translate js to typescript
- add the sequelize ORM for more convenience when interacting with the sqlite database
- enable hierarchical folder structure for storing documents
- remove eleventy
parent cca1e447
No related branches found
No related tags found
No related merge requests found
Pipeline #91098 passed with stages
in 1 minute and 49 seconds
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1904 additions and 4356 deletions
eleventy-site/_includes/layouts/default.njk deleted 100644 → 0
+ 0
− 12
View file @ cca1e447
<!doctype html>
<html lang="en" dir="ltr">
<head>
<meta name="description" content="A list of projects linked with the Ensembl projects">
<link href="/css/styles.css" rel="stylesheet" type="text/css">
</head>
<body>
<main>
{{ content | safe }}
</main>
</body>
</html>
eleventy-site/articles.njk deleted 100644 → 0
+ 0
− 25
View file @ cca1e447
---
pagination:
data: articles
size: 1
alias: article
permalink: "articles/{{ article.data.slug }}/"
layout: default
---
{% if article.data.relatedVideo %}
<div class="article-with-video-container">
<div>
{{ article.body | safe }}
</div>
<div>
<h1>Related video</h1>
<iframe width="560" height="315" src="{{ article.data.relatedVideo.data.url }}" frameborder="0" allowfullscreen></iframe>
{{ article.data.relatedVideo.body | safe }}
</div>
</div>
{% else %}
<div class="simple-article-container">
{{ article.body | safe }}
</div>
{% endif %}
eleventy-site/css/styles.css deleted 100644 → 0
+ 0
− 19
View file @ cca1e447
a, a:visited, a:focus {
color: blue;
}
.index-container {
max-width: 800px;
margin: auto;
}
.simple-article-container {
max-width: 1000px;
margin: auto;
}
.article-with-video-container {
display: grid;
grid-template-columns: 1.5fr 1fr;
grid-column-gap: 1rem;
}
eleventy-site/index.njk deleted 100644 → 0
+ 0
− 19
View file @ cca1e447
---
layout: default
---
<div class="index-container">
<h1>
Available articles
</h1>
<ul>
{% for article in articles %}
<li>
<a href="/articles/{{ article.data.slug }}">{{ article.data.slug }}</a>
</li>
{% else %}
<li>There are no articles</li>
{% endfor %}
</ul>
</div>
k8s/deployment.yaml
+ 4
− 4
View file @ dc85a0bc
apiVersion: apps/v1beta1 apiVersion: apps/v1beta1
kind: Deployment kind: Deployment
metadata: metadata:
name: ensembl-docs-prototype-deployment name: ensembl-help-and-docs-deployment
spec: spec:
replicas: 1 replicas: 1
template: template:
metadata: metadata:
labels: labels:
app: ensembl-docs-prototype app: ensembl-help-and-docs
spec: spec:
imagePullSecrets: imagePullSecrets:
- name: ensembl-docs-prototype-secret - name: ensembl-help-and-docs-pull-secret
containers: containers:
- name: ensembl-docs-prototype - name: ensembl-help-and-docs
image: <DOCKER_IMAGE> image: <DOCKER_IMAGE>
command: [ "npm", "run", "start-server" ] command: [ "npm", "run", "start-server" ]
ports: ports:
......
k8s/service-node.yaml
+ 3
− 3
View file @ dc85a0bc
apiVersion: v1 apiVersion: v1
kind: Service kind: Service
metadata: metadata:
name: ensembl-docs-prototype-svc name: ensembl-help-and-docs-svc
labels: labels:
app: ensembl-docs-prototype-svc app: ensembl-help-and-docs-svc
spec: spec:
selector: selector:
app: ensembl-docs-prototype app: ensembl-help-and-docs
type: NodePort type: NodePort
ports: ports:
- port: 8000 - port: 8000
......
package-lock.json
+ 1859
− 3938
View file @ dc85a0bc
This diff is collapsed.
package.json
+ 38
− 20
View file @ dc85a0bc
{ {
"name": "markdown-server", "name": "ensembl-help-and-docs-api",
"version": "1.0.0", "version": "1.0.0",
"description": "", "description": "",
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"clean": "rm -rf ./build _site", "clean": "rm -rf ./build",
"prebuild": "npm run clean", "dev": "ts-node-dev --respawn --transpileOnly src/server/index.ts",
"build": "mkdirp build && node ./build.js && npm run eleventy-build", "prebuild": "npm run clean && mkdirp build",
"start-server": "node server/server.js", "build": "npm run build-docs && npm run build-server",
"eleventy-clean": "rm -rf _site", "build-docs": "NODE_ENV=build-docs ts-node ./src/build-scripts/index.ts",
"start-eleventy": "npm run eleventy-clean && eleventy --serve", "build-server": "tsc",
"eleventy-build": "eleventy" "start-server": "NODE_ENV=production node build/src/server/index.js"
}, },
"keywords": [], "keywords": [],
"author": "", "author": "",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"@11ty/eleventy": "0.10.0", "cors": "2.8.5",
"cross-fetch": "3.0.4", "cross-fetch": "3.0.5",
"express": "4.17.1", "express": "4.17.1",
"fs-extra": "9.0.1",
"globby": "11.0.1",
"lodash": "4.17.19",
"lunr": "2.3.8", "lunr": "2.3.8",
"sequelize": "6.3.4",
"sqlite3": "5.0.0"
},
"devDependencies": {
"@types/bluebird": "3.5.32",
"@types/cors": "2.8.7",
"@types/express": "4.17.7",
"@types/fs-extra": "9.0.1",
"@types/lodash": "4.14.159",
"@types/sqlite3": "3.1.6",
"@types/unist": "2.0.3",
"@types/validator": "13.1.0",
"mkdirp": "1.0.4", "mkdirp": "1.0.4",
"nodemon": "2.0.4",
"rehype-raw": "4.0.2", "rehype-raw": "4.0.2",
"rehype-stringify": "7.0.0", "rehype-stringify": "8.0.0",
"remark-extract-frontmatter": "2.0.2", "remark-extract-frontmatter": "2.0.3",
"remark-frontmatter": "2.0.0", "remark-frontmatter": "2.0.0",
"remark-parse": "8.0.0", "remark-parse": "8.0.3",
"remark-rehype": "6.0.0", "remark-rehype": "7.0.0",
"remark-stringify": "8.0.0", "remark-stringify": "8.1.1",
"sqlite": "4.0.6",
"sqlite3": "4.1.1",
"strip-markdown": "3.1.2", "strip-markdown": "3.1.2",
"to-vfile": "6.1.0", "to-vfile": "6.1.0",
"unified": "9.0.0", "ts-node": "8.10.2",
"unist-util-visit": "2.0.2", "ts-node-dev": "1.0.0-pre.48",
"yaml": "1.8.3" "tsconfig-paths": "3.9.0",
"typescript": "3.9.7",
"unified": "9.1.0",
"unist-util-visit": "2.0.3",
"yaml": "1.10.0"
} }
} }
scripts/build-database/addArticles.js deleted 100644 → 0
+ 0
− 44
View file @ cca1e447
const { addArticleTags } = require('./addTags');
const addArticles = async (db, articles) => {
for (article of articles) {
article = addParents(article, articles);
await insertArticle(db, article);
await addArticleTags(db, article);
}
};
const insertArticle = async (db, fileData) => {
const { html, ...otherFields } = fileData;
const sql = `INSERT INTO articles(filename, body, data) VALUES (:filename, :body, :data)`;
await db.run(sql, {
':filename': fileData.slug,
':body': html,
':data': JSON.stringify(otherFields)
});
};
const addParents = (article, articles) => {
if (!article.parent) {
return article;
}
return buildParents(article, articles);
};
const buildParents = (article, articles, parent) => {
if (!article.parents) {
article.parents = [];
}
const nextParentName = parent ? parent.parent : article.parent;
const nextParent = articles.find(({ slug }) => slug === nextParentName);
if (!nextParent) {
return article;
} else {
article = Object.assign({}, article, { parents: [...article.parents, nextParent.slug] });
return buildParents(article, articles, nextParent);
}
};
module.exports = addArticles;
scripts/build-database/addTags.js deleted 100644 → 0
+ 0
− 53
View file @ cca1e447
const addArticleTags = async (db, article) => {
const { slug, tags } = article;
if (!tags) {
return;
}
await addTags(db, tags);
for (tag of tags) {
const { id: savedArticleId } = await db.get('SELECT id FROM articles WHERE filename = ?', slug);
const { id: savedTagId } = await db.get('SELECT id FROM tags WHERE name = ?', tag);
await db.run('INSERT INTO articles_tags (article_id, tag_id) VALUES (:articleId, :tagId)', {
':articleId': savedArticleId,
':tagId': savedTagId
});
}
};
const addVideoTags = async (db, video) => {
const { slug, tags } = video;
if (!tags) {
return;
}
await addTags(db, tags);
for (tag of tags) {
const { id: savedVideoId } = await db.get('SELECT id FROM videos WHERE filename = ?', slug);
const { id: savedTagId } = await db.get('SELECT id FROM tags WHERE name = ?', tag);
await db.run('INSERT INTO videos_tags (video_id, tag_id) VALUES (:video_id, :tagId)', {
':video_id': savedVideoId,
':tagId': savedTagId
});
}
};
const addTags = async (db, tags) => {
for (tag of tags) {
await addTag(db, tag);
}
}
const addTag = async (db, tag) => {
const result = await db.get('SELECT id FROM tags WHERE name = ?', tag);
const { id: savedTagId } = result || {};
if (!savedTagId) {
await db.run('INSERT INTO tags (name) VALUES (?)', tag);
}
};
module.exports = {
addArticleTags,
addVideoTags
};
scripts/build-database/addVideos.js deleted 100644 → 0
+ 0
− 20
View file @ cca1e447
const { addVideoTags } = require('./addTags');
const addVideos = async (db, videos) => {
for (video of videos) {
await insertVideo(db, video);
await addVideoTags(db, video);
}
};
const insertVideo = async (db, video) => {
const { html, ...otherFields } = video;
const sql = `INSERT INTO videos(filename, body, data) VALUES (:filename, :body, :data)`;
await db.run(sql, {
':filename': video.slug,
':body': html,
':data': JSON.stringify(otherFields)
});
};
module.exports = addVideos;
scripts/build-database/createDatabase.js deleted 100644 → 0
+ 0
− 45
View file @ cca1e447
const sqlite = require('sqlite');
const sqlite3 = require('sqlite3');
const sql = `
CREATE TABLE articles (
id integer PRIMARY KEY,
filename text NOT NULL,
body text NOT NULL,
data text
);
CREATE TABLE videos (
id integer PRIMARY KEY,
filename text NOT NULL,
body text NOT NULL,
data text
);
CREATE TABLE tags (
id integer PRIMARY KEY,
name text NOT NULL
);
CREATE TABLE articles_tags (
id integer PRIMARY KEY,
article_id text NOT NULL,
tag_id integer NOT NULL
);
CREATE TABLE videos_tags (
id integer PRIMARY KEY,
video_id text NOT NULL,
tag_id integer NOT NULL
);
`;
const config = require('../../config');
const createDatabase = async () => {
sqlite3.verbose();
const db = await sqlite.open({
filename: config.databasePath,
driver: sqlite3.Database
});
await db.exec(sql);
return db;
};
module.exports = createDatabase;
scripts/build-database/markdownImagePlugin.js deleted 100644 → 0
+ 0
− 19
View file @ cca1e447
const visit = require('unist-util-visit');
const attacher = () => {
const transformer = (tree, file) => {
visit(tree, 'image', imageVisitor);
};
return transformer;
};
const imageVisitor = (node) => {
let host;
if (process.env.NODE_ENV === 'production') {
host = 'http://193.62.55.158:30799';
node.url = `${host}${node.url}`;
}
};
module.exports = attacher;
scripts/build-database/readSourceFiles.js deleted 100644 → 0
+ 0
− 29
View file @ cca1e447
const fs = require('fs');
const path = require('path');
const config = require('../../config');
const parseMarkdown = require('./parseMarkdown');
const readSourceFiles = async () => {
const { articlesPath, videosPath } = config;
const articles = await readFilesFromDir(articlesPath);
const videos = await readFilesFromDir(videosPath);
return {
articles,
videos
};
};
const readFilesFromDir = async (dir) => {
const promisedData = fs.readdirSync(dir)
.map(async fileName => {
const slug = fileName.replace(/\.md$/, '');
const filePath = path.join(dir, fileName);
const fileData = await parseMarkdown(filePath);
return Object.assign({}, fileData, { slug });
});
return await Promise.all(promisedData);
};
module.exports = readSourceFiles;
scripts/build-index/readSourceFiles.js deleted 100644 → 0
+ 0
− 27
View file @ cca1e447
const fs = require('fs');
const path = require('path');
const config = require('../../config');
const parseFile = require('./parseFile');
const readSourceFiles = async () => {
const { articlesPath } = config;
const articles = await readFilesFromDir(articlesPath);
return {
articles
};
};
const readFilesFromDir = async (dir) => {
const promisedData = fs.readdirSync(dir)
.map(async fileName => {
const slug = fileName.replace(/\.md$/, '');
const filePath = path.join(dir, fileName);
const fileData = await parseFile(filePath);
return Object.assign({}, fileData, { slug });
});
return await Promise.all(promisedData);
};
module.exports = readSourceFiles;
scripts/get-from-database/getArticle.js deleted 100644 → 0
+ 0
− 25
View file @ cca1e447
const getVideo = require('./getVideo');
const getArticle = async (db, slug) => {
try {
const match = await db.get('SELECT * FROM articles WHERE filename = ?', slug);
if (!match) {
return;
}
match.data = JSON.parse(match.data);
const relatedVideoSlug = match.data['related-video'];
if (relatedVideoSlug) {
const relatedVideo = await getVideo(db, relatedVideoSlug);
match.data.relatedVideo = relatedVideo;
}
return match;
} catch (error) {
console.log('error getting the document', error);
}
};
module.exports = getArticle;
scripts/get-from-database/getVideo.js deleted 100644 → 0
+ 0
− 15
View file @ cca1e447
const getVideo = async (db, slug) => {
try {
const match = await db.get('SELECT * FROM videos WHERE filename = ?', slug);
if (!match) {
return;
}
match.data = JSON.parse(match.data);
return match;
} catch (error) {
console.log('error getting the document', error);
}
};
module.exports = getVideo;
scripts/getFromDatabase.js deleted 100644 → 0
+ 0
− 17
View file @ cca1e447
const getOneDocument = async (db, name) => {
try {
const match = await db.get('SELECT * FROM documents WHERE name = ?', name);
if (!match) {
return;
}
match.data = JSON.parse(match.data);
return match;
} catch (error) {
console.log('error getting the document', error);
}
};
module.exports = {
getOneDocument
};
scripts/searchIndex.js deleted 100644 → 0
+ 0
− 9
View file @ cca1e447
const path = require('path');
const lunr = require('lunr');
const searchIndex = (query, index) => {
const idx = lunr.Index.load(index);
return idx.search(query);
};
module.exports = searchIndex;
server/app.js deleted 100644 → 0
+ 0
− 13
View file @ cca1e447
const path = require('path');
const express = require('express');
const app = express();
const getArticle = require('../api/article');
const search = require('../api/search');
app.use(express.static(path.join(__dirname, '../_site')));
app.use('/api/article', getArticle);
app.use('/api/search', search);
module.exports = app;
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment