From c2cf7fc07d56c0855fbdc3c8770bf10cecf4ec00 Mon Sep 17 00:00:00 2001
From: Yuci Gou <yuci.gou@gmail.com>
Date: Tue, 5 Mar 2019 17:49:08 +0000
Subject: [PATCH] Use the express-middleware-minio package
---
config/default.js | 1 +
package.json | 4 +-
server/ftp-integration/api.js | 2 +-
server/minio-server/src/FileBackend.js | 18 +-
.../pubsweet-component-minio/README.md | 87 ----------
.../modules/pubsweet-component-minio/index.js | 153 -----------------
.../pubsweet-component-minio/minio-client.js | 160 ------------------
.../pubsweet-component-minio/package.json | 28 ---
.../modules/pubsweet-component-minio/utils.js | 18 --
server/pdf-conversion/api.js | 2 +-
server/utils/files.js | 2 +-
.../xpub-server/entities/file/uploadFiles.js | 2 +-
12 files changed, 16 insertions(+), 461 deletions(-)
delete mode 100755 server/modules/pubsweet-component-minio/README.md
delete mode 100755 server/modules/pubsweet-component-minio/index.js
delete mode 100755 server/modules/pubsweet-component-minio/minio-client.js
delete mode 100755 server/modules/pubsweet-component-minio/package.json
delete mode 100755 server/modules/pubsweet-component-minio/utils.js
diff --git a/config/default.js b/config/default.js
index 959f17f44..5a349e22f 100755
--- a/config/default.js
+++ b/config/default.js
@@ -44,6 +44,7 @@ module.exports = {
sendmail: true,
},
},
+ logger,
'pubsweet-server': {
db: {
port: 5432,
diff --git a/package.json b/package.json
index 9abe8a320..a2e3a9044 100755
--- a/package.json
+++ b/package.json
@@ -15,7 +15,7 @@
"dependencies": {
"@pubsweet/base-model": "^1.0.9",
"@pubsweet/component-send-email": "^0.2.4",
- "@pubsweet/db-manager":"^2.2.3",
+ "@pubsweet/db-manager": "^2.2.3",
"@pubsweet/logger": "^0.2.3",
"@pubsweet/ui": "9.0.0",
"@pubsweet/ui-toolkit": "2.0.0",
@@ -31,6 +31,7 @@
"download": "^7.1.0",
"email-validator": "^2.0.4",
"es6-promise": "^4.2.4",
+ "express-middleware-minio": "^2.0.0",
"faker": "^4.1.0",
"font-awesome": "^4.7.0",
"form-data": "^2.3.2",
@@ -48,7 +49,6 @@
"loadable-components": "^0.3.0",
"md5-file": "^4.0.0",
"mime-types": "^2.1.21",
- "minio": "^7.0.1",
"moment": "^2.18.1",
"node-cron": "^2.0.3",
"node-fetch": "^2.2.0",
diff --git a/server/ftp-integration/api.js b/server/ftp-integration/api.js
index 57643637c..10e620bd4 100644
--- a/server/ftp-integration/api.js
+++ b/server/ftp-integration/api.js
@@ -6,7 +6,7 @@ const { createHttpLink } = require('apollo-link-http')
const gql = require('graphql-tag')
const fetch = require('node-fetch')
const uuidv4 = require('uuid/v4')
-const { minioClient } = require('../modules/pubsweet-component-minio')
+const { minioClient } = require('express-middleware-minio')
const Manuscript = require('../xpub-model/entities/manuscript/data-access')
const ManuscriptManager = require('../xpub-model/entities/manuscript/index')
const Journal = require('../xpub-model/entities/journal/data-access')
diff --git a/server/minio-server/src/FileBackend.js b/server/minio-server/src/FileBackend.js
index cda962abd..666f38714 100755
--- a/server/minio-server/src/FileBackend.js
+++ b/server/minio-server/src/FileBackend.js
@@ -1,15 +1,15 @@
const config = require('config')
const rfr = require('rfr')
-const minioClient = rfr('server/modules/pubsweet-component-minio')
+const expressMinio = require('express-middleware-minio')
-const minioMiddleware = minioClient.middleware()
+const minioMiddleware = expressMinio.middleware()
const download_path = config.file.url.download
module.exports = app => {
app.post(
'/api/upload',
- minioMiddleware({ op: minioClient.Ops.post }),
+ minioMiddleware({ op: expressMinio.Ops.post }),
(req, res) => {
if (req.minio.error) {
res.status(400).json({ error: req.minio.error })
@@ -21,7 +21,7 @@ module.exports = app => {
app.get(
'/api/files',
- minioMiddleware({ op: minioClient.Ops.list }),
+ minioMiddleware({ op: expressMinio.Ops.list }),
(req, res) => {
if (req.minio.error) {
res.status(400).json({ error: req.minio.error })
@@ -33,7 +33,7 @@ module.exports = app => {
app.get(
`/api/files/:filename`,
- minioMiddleware({ op: minioClient.Ops.get }),
+ minioMiddleware({ op: expressMinio.Ops.get }),
(req, res) => {
if (req.minio.error) {
res.status(400).json({ error: req.minio.error })
@@ -44,14 +44,14 @@ module.exports = app => {
if (err) {
res.status(500).json({ err })
}
- minioClient.utils.removeFile(req.minio.get.path)
+ expressMinio.utils.removeFile(req.minio.get.path)
})
},
)
app.delete(
'/api/files/:filename',
- minioMiddleware({ op: minioClient.Ops.delete }),
+ minioMiddleware({ op: expressMinio.Ops.delete }),
(req, res) => {
if (req.minio.error) {
res.status(400).json({ error: req.minio.error })
@@ -65,7 +65,7 @@ module.exports = app => {
* files for the pdf conversion stage. */
app.get(
`${download_path}/:filename`,
- minioMiddleware({ op: minioClient.Ops.get }),
+ minioMiddleware({ op: expressMinio.Ops.get }),
(req, res) => {
if (req.minio.error) {
res.status(400).json({ error: req.minio.error })
@@ -76,7 +76,7 @@ module.exports = app => {
if (err) {
console.warn('Download failed: ', err)
}
- minioClient.utils.removeFile(req.minio.get.path)
+ expressMinio.utils.removeFile(req.minio.get.path)
})
},
)
diff --git a/server/modules/pubsweet-component-minio/README.md b/server/modules/pubsweet-component-minio/README.md
deleted file mode 100755
index 34d53ee11..000000000
--- a/server/modules/pubsweet-component-minio/README.md
+++ /dev/null
@@ -1,87 +0,0 @@
-# Introduction
-
-This Minio middleware is written for Node.js Express apps to use Minio to store files.
-
-Files can be uploaded to a predefined folder in a predefined bucket in a Minio sever. Once a file has been uploaded successfully, you will be informed of the filename known to Minio for this file. Later on, you can use this filename to download or delete the file.
-
-The Minio middleware also allows you to list all files stored inside the predefined folder in the predefined bucket in Minio.
-
-# How to use the package?
-
-## First of all, install the package:
-
-```shell
-npm i express-middleware-minio
-```
-
-## Second, you need to add .env to get it up running, e.g.:
-
-```shell
-MINIO_ACCESS_KEY=xxx
-MINIO_SECRET_KEY=xxx
-MINIO_ENDPOINT='192.111.111.131'
-MINIO_PORT=9000
-MINIO_SECURITY=false
-MINIO_BUCKET=manuscripts
-MINIO_UPLOADS_FOLDER_NAME=uploads
-```
-
-## Then you can use the Minio middleware in your Express application:
-Four operations are provided:
-* post
-* get
-* delete
-* list
-
-You can use them the following way:
-```javascript
-const rfr = require('rfr')
-const minioClient = rfr('server/modules/pubsweet-component-minio')
-console.log(minioClient.Ops.post)
-```
-
-You can find below an example.
-
-```javascript
-const rfr = require('rfr')
-const minioClient = rfr('server/modules/pubsweet-component-minio')
-const minioMiddleware = minioClient.middleware();
-
-app.post('/api/files', minioMiddleware({op: minioClient.Ops.post}), (req, res) => {
- if (req.minio.error) {
- res.status(400).json({ error: req.minio.error })
- }
-
- res.send({ filename: req.minio.post.filename })
-})
-
-app.get('/api/files',
- minioMiddleware({op: minioClient.Ops.list}),
- (req, res) => {
- if (req.minio.error) {
- res.status(400).json({ error: req.minio.error })
- }
- res.send(req.minio.list);
- }
-)
-
-app.get('/api/files/:filename',
- minioMiddleware({op: minioClient.Ops.get}),
- (req, res) => {
- if (req.minio.error) {
- res.status(400).json({ error: req.minio.error })
- }
- res.download(req.minio.get);
- }
-)
-
-app.delete('/api/files/:filename',
- minioMiddleware({op: minioClient.Ops.delete}),
- (req, res) => {
- if (req.minio.error) {
- res.status(400).json({ error: req.minio.error })
- }
- res.send(req.minio.delete);
- }
-)
-```
diff --git a/server/modules/pubsweet-component-minio/index.js b/server/modules/pubsweet-component-minio/index.js
deleted file mode 100755
index bc134298d..000000000
--- a/server/modules/pubsweet-component-minio/index.js
+++ /dev/null
@@ -1,153 +0,0 @@
-require('dotenv').config()
-const formidable = require('formidable')
-const uuidv1 = require('uuid/v1')
-const logger = require('@pubsweet/logger')
-const minioClient = require('./minio-client.js')
-const utils = require('./utils')
-
-const Ops = Object.freeze({ post: 1, list: 2, get: 3, delete: 4 })
-
-const extractFileExtension = filename => {
- if (filename) {
- return filename.split('.').pop()
- }
-
- return ''
-}
-
-const validityCheck = (req, options) => {
- if (!options) {
- const error = 'Options not provided'
- req.minio = { error }
- return false
- }
-
- let supported = false
- Object.keys(Ops).forEach(key => {
- if (Ops[key] === options.op) {
- supported = true
- }
- })
-
- if (!supported) {
- const error = 'Operation not supported'
- req.minio = { error }
- return false
- }
-
- return true
-}
-
-const handlePost = (req, next, fields, files) => {
- let filename = uuidv1()
- if (files.file && files.file.name) {
- const extension = extractFileExtension(files.file.name)
- if (extension) {
- filename += `.${extension}`
- }
- }
-
- minioClient.uploadFile(
- filename,
- (files.file && files.file.name) || '',
- (files.file && files.file.type) || '',
- (files.file && files.file.path) || '',
- (error, etag) => {
- if (error) {
- req.minio = { error }
- } else {
- req.minio = { post: { filename: `${filename}`, etag } }
- }
- next()
- },
- )
-}
-
-const handleList = (req, next) => {
- minioClient.listFiles((error, list) => {
- if (error) {
- req.minio = { error }
- } else {
- req.minio = { list }
- }
- next()
- })
-}
-
-const handleGet = async (req, next) => {
- let stat
- try {
- stat = await minioClient.getFileStat(req.params.filename)
- } catch (error) {
- logger.error('minio handleGet error: ', error)
- req.minio = { error }
- next()
- return
- }
-
- const tmpFile = utils.getTempPath(req.params.filename)
- minioClient.getFile(req.params.filename, tmpFile, error => {
- if (error) {
- req.minio = { error }
- } else {
- let fielname = req.params.filename
- if (stat.metaData && stat.metaData['file-name']) {
- fielname = stat.metaData['file-name']
- }
- req.minio = {
- get: {
- path: tmpFile,
- originalName: fielname,
- },
- }
- }
- next()
- })
-}
-
-const handleDelete = (req, next) => {
- minioClient.deleteFile(req.params.filename, error => {
- if (error) {
- req.minio = { error }
- } else {
- req.minio = { delete: 'Success' }
- }
- next()
- })
-}
-
-const handleRequests = (req, next, options) => {
- if (!validityCheck(req, options)) {
- next()
- return
- }
-
- if (options.op === Ops.post) {
- const form = new formidable.IncomingForm()
- form.parse(req, (err, fields, files) => {
- if (err) {
- req.minio = { error: err }
- next()
- } else {
- handlePost(req, next, fields, files)
- }
- })
- } else if (options.op === Ops.list) {
- handleList(req, next)
- } else if (options.op === Ops.get) {
- handleGet(req, next)
- } else if (options.op === Ops.delete) {
- handleDelete(req, next)
- }
-}
-
-module.exports = {
- Ops,
- utils,
- minioClient,
- middleware() {
- return options => (req, res, next) => {
- handleRequests(req, next, options)
- }
- },
-}
diff --git a/server/modules/pubsweet-component-minio/minio-client.js b/server/modules/pubsweet-component-minio/minio-client.js
deleted file mode 100755
index 57968e8fd..000000000
--- a/server/modules/pubsweet-component-minio/minio-client.js
+++ /dev/null
@@ -1,160 +0,0 @@
-const Minio = require('minio')
-const logger = require('@pubsweet/logger')
-
-const { MINIO_UPLOADS_FOLDER_NAME, MINIO_BUCKET } = process.env
-
-function sleep(ms) {
- return new Promise(resolve => {
- setTimeout(resolve, ms)
- })
-}
-
-const MinioClientClass = (() => {
- let minioClient
- let bucketCreationInProcess = false
- let bucketExists
-
- const initBucket = async minioClient => {
- let exists
- try {
- exists = await minioClient.bucketExists(MINIO_BUCKET)
- } catch (err) {
- logger.error('initBucket - bucketExists: ', err)
- return false
- }
- if (exists) {
- logger.debug('initBucket: bucket exists', MINIO_BUCKET)
- bucketExists = true
- return true
- }
-
- if (bucketCreationInProcess) {
- await sleep(10000)
- }
-
- if (!bucketExists) {
- bucketCreationInProcess = true
- try {
- await minioClient.makeBucket(MINIO_BUCKET, 'eu-west-2')
- } catch (err) {
- bucketCreationInProcess = false
- logger.error('initBucket - makeBucket: ', err)
- return false
- }
- bucketExists = true
- bucketCreationInProcess = false
- logger.info('initBucket: bucket created', MINIO_BUCKET)
- }
-
- return true
- }
-
- const getInstance = async () => {
- if (minioClient && bucketExists) {
- return minioClient
- }
-
- minioClient = new Minio.Client({
- endPoint: process.env.MINIO_ENDPOINT,
- port: Number(process.env.MINIO_PORT),
- useSSL: process.env.MINIO_SECURITY === 'true',
- accessKey: process.env.MINIO_ACCESS_KEY,
- secretKey: process.env.MINIO_SECRET_KEY,
- })
-
- await initBucket(minioClient)
- return minioClient
- }
-
- return {
- getInstance,
- }
-})()
-
-module.exports = {
- async uploadFile(filename, oriFilename, fileType, tempFilePath, callback) {
- const uploads = MINIO_UPLOADS_FOLDER_NAME
- const filePath = `${uploads}/${filename}`
-
- const metaData = {
- 'content-type': fileType,
- 'file-name': oriFilename,
- }
-
- const minioClient = await MinioClientClass.getInstance()
- minioClient.fPutObject(
- MINIO_BUCKET,
- filePath,
- tempFilePath,
- metaData,
- callback,
- )
- },
-
- async uploadFileSteam(filename, oriFilename, fileType, fileStream, callback) {
- const uploads = MINIO_UPLOADS_FOLDER_NAME
- const filePath = `${uploads}/${filename}`
-
- const metaData = {
- 'content-type': fileType,
- 'file-name': oriFilename,
- }
-
- const minioClient = await MinioClientClass.getInstance()
- minioClient.putObject(
- MINIO_BUCKET,
- filePath,
- fileStream,
- metaData,
- callback,
- )
- },
-
- async listFiles(callback) {
- const uploads = MINIO_UPLOADS_FOLDER_NAME
- const prefix = `${uploads}`
-
- const minioClient = await MinioClientClass.getInstance()
- const stream = minioClient.listObjects(MINIO_BUCKET, prefix, true)
- const list = []
- stream.on('data', obj => {
- list.push(obj)
- })
- stream.on('error', err => {
- callback(err)
- })
- stream.on('end', () => {
- callback(null, list)
- })
- },
-
- async getFile(fileName, tmpFile, callback) {
- const uploads = MINIO_UPLOADS_FOLDER_NAME
- const objectName = `${uploads}/${fileName}`
- const minioClient = await MinioClientClass.getInstance()
- minioClient.fGetObject(MINIO_BUCKET, objectName, tmpFile, callback)
- },
-
- getFileStat(filename) {
- return new Promise(async (resolve, reject) => {
- const uploads = MINIO_UPLOADS_FOLDER_NAME
- const objectName = `${uploads}/${filename}`
- const minioClient = await MinioClientClass.getInstance()
- minioClient.statObject(MINIO_BUCKET, objectName, (err, stat) => {
- if (err) {
- return reject(err)
- }
- resolve(stat)
- })
- })
- },
-
- async deleteFile(fileName, callback) {
- const uploads = MINIO_UPLOADS_FOLDER_NAME
- const objectName = `${uploads}/${fileName}`
- const minioClient = await MinioClientClass.getInstance()
- minioClient.removeObject(MINIO_BUCKET, objectName, err => {
- callback(err)
- })
- },
-}
diff --git a/server/modules/pubsweet-component-minio/package.json b/server/modules/pubsweet-component-minio/package.json
deleted file mode 100755
index 4ad81a38c..000000000
--- a/server/modules/pubsweet-component-minio/package.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "express-middleware-minio",
- "version": "1.2.1",
- "description": "An Express middleware that helps you store files in Minio.",
- "keywords": [
- "express",
- "middleware",
- "minio"
- ],
- "main": "index.js",
- "scripts": {
- "test": "echo \"Error: no test specified\" && exit 1",
- "lint": "eslint ."
- },
- "author": "Yuci Gou <yuci.gou@gmail.com>",
- "license": "ISC",
- "dependencies": {
- "dotenv": "^6.0.0",
- "formidable": "^1.2.1",
- "minio": "^6.0.0",
- "uuid": "^3.2.1"
- },
- "devDependencies": {
- "eslint": "^4.19.1",
- "eslint-config-airbnb-base": "^13.0.0",
- "eslint-plugin-import": "^2.13.0"
- }
-}
diff --git a/server/modules/pubsweet-component-minio/utils.js b/server/modules/pubsweet-component-minio/utils.js
deleted file mode 100755
index ff8fa5582..000000000
--- a/server/modules/pubsweet-component-minio/utils.js
+++ /dev/null
@@ -1,18 +0,0 @@
-const fs = require('fs')
-const uuidv1 = require('uuid/v1')
-
-const temp_dir = '/tmp'
-const getTempPath = filename => `${temp_dir}/${filename}.${uuidv1()}`
-
-const removeFile = tmpFile => {
- fs.unlink(tmpFile, err => {
- if (err) {
- console.warn(`Error deleting file ${tmpFile}: `, err)
- }
- })
-}
-
-module.exports = {
- getTempPath,
- removeFile,
-}
diff --git a/server/pdf-conversion/api.js b/server/pdf-conversion/api.js
index f5aa4b79f..fadc3963c 100644
--- a/server/pdf-conversion/api.js
+++ b/server/pdf-conversion/api.js
@@ -3,7 +3,7 @@ const fs = require('fs')
const uuidv4 = require('uuid/v4')
require('dotenv').config()
const https = require('https')
-const { minioClient } = require('../modules/pubsweet-component-minio')
+const { minioClient } = require('express-middleware-minio')
const config = require('config')
const fetch = require('node-fetch')
// const ManuscriptManager = require('../xpub-model/entities/manuscript')
diff --git a/server/utils/files.js b/server/utils/files.js
index c6c3f8d36..f9e65e6f2 100644
--- a/server/utils/files.js
+++ b/server/utils/files.js
@@ -8,7 +8,7 @@ const uuidv4 = require('uuid/v4')
const download = require('download')
const fetch = require('node-fetch')
-const { minioClient } = rfr('server/modules/pubsweet-component-minio')
+const { minioClient } = require('express-middleware-minio')
module.exports.getManifestFilename = function getManifestFilename(tmpPath) {
return new Promise((resolve, reject) => {
diff --git a/server/xpub-server/entities/file/uploadFiles.js b/server/xpub-server/entities/file/uploadFiles.js
index df2aa00e2..5db2a1217 100644
--- a/server/xpub-server/entities/file/uploadFiles.js
+++ b/server/xpub-server/entities/file/uploadFiles.js
@@ -3,7 +3,7 @@ const logger = require('@pubsweet/logger')
const config = require('config')
const rfr = require('rfr')
-const { minioClient } = rfr('server/modules/pubsweet-component-minio')
+const { minioClient } = require('express-middleware-minio')
const FileModel = rfr('server/xpub-model/entities/file/data-access')
const extractFileExtension = filename => {
--
GitLab