From 164cf6fd5d4fefa859730d836595abe61a5fa21b Mon Sep 17 00:00:00 2001
From: Yogmatee Roochun <yroochun@ebi.ac.uk>
Date: Fri, 15 Mar 2019 14:03:15 +0000
Subject: [PATCH] email - ongoing
---
package.json | 1 +
server/email/index.js | 8 +++++++
server/email/templates/ftp.js | 6 +++++
server/email/templates/index.js | 2 ++
server/ftp-integration/api.js | 5 +++-
server/ftp-integration/taggedXML.js | 28 ++++++++++++++++++++--
server/ftp-integration/toTaggers.js | 36 ++++++++++++++++++++++-------
server/utils/files.js | 11 +++++++++
yarn.lock | 2 +-
9 files changed, 87 insertions(+), 12 deletions(-)
diff --git a/package.json b/package.json
index d76ad7013..abd77d4a9 100755
--- a/package.json
+++ b/package.json
@@ -89,6 +89,7 @@
"request": "^2.88.0",
"request-promise": "^4.2.4",
"rfr": "^1.2.3",
+ "rimraf": "^2.6.3",
"supertest": "^3.0.0",
"tar": "^4.4.8",
"tar-fs": "^1.16.3",
diff --git a/server/email/index.js b/server/email/index.js
index fccad8b35..12a41d9b0 100755
--- a/server/email/index.js
+++ b/server/email/index.js
@@ -10,6 +10,7 @@ const {
removeDupeTemplate,
newPackageForTaggingTemplate,
processedTaggingFilesTemplate,
+ errorDevTemplate,
bulkUploadTemplate,
manuscriptHasBeenPublishedTemplate,
finalReviewTemplate,
@@ -64,6 +65,12 @@ const processedTaggerEmail = (email, manId, title, packageName) => {
sendMail(email, subject, html, null, null, system)
}
+const errorDevEmail = (email, err) => {
+ const html = errorDevTemplate(err)
+ const subject = 'Something went wrong with the FTP utility functions'
+ sendMail(email, subject, html, null, null, system)
+}
+
const bulkUploaderEmail = (email, message) => {
const html = bulkUploadTemplate(message)
const subject = 'Error while processing package'
@@ -159,6 +166,7 @@ module.exports = {
userMessage,
taggerEmail,
processedTaggerEmail,
+ errorDevEmail,
taggerErrorEmail,
bulkUploaderEmail,
manuscriptHasBeenPublishedEmail,
diff --git a/server/email/templates/ftp.js b/server/email/templates/ftp.js
index 2fd0b769b..bc731c4d0 100644
--- a/server/email/templates/ftp.js
+++ b/server/email/templates/ftp.js
@@ -14,6 +14,11 @@ const processedTaggingFilesTemplate = (manId, title, packageName) => `
<p>Kind regards,</p>
<p>The Europe PMC plus system</p>
`
+const errorDevTemplate = err => `
+ <p>Something went wrong with the FTP utility.<p>
+ <p>Error: <b>${err}</b>
+ <p>The Europe PMC plus system</p>
+ `
const bulkUploadTemplate = message => `
<p>There was an error while processing the package: <b>${message}</b><p>
<p>Kind regards,</p>
@@ -32,4 +37,5 @@ module.exports = {
bulkUploadTemplate,
manuscriptHasBeenPublishedTemplate,
processedTaggingFilesTemplate,
+ errorDevTemplate,
}
diff --git a/server/email/templates/index.js b/server/email/templates/index.js
index c05045c08..8e0f62905 100644
--- a/server/email/templates/index.js
+++ b/server/email/templates/index.js
@@ -9,6 +9,7 @@ const { removeDupeTemplate } = require('./removeDuplicate')
const {
newPackageForTaggingTemplate,
processedTaggingFilesTemplate,
+ errorDevTemplate,
bulkUploadTemplate,
manuscriptHasBeenPublishedTemplate,
} = require('./ftp')
@@ -84,6 +85,7 @@ module.exports = {
finalReviewTemplate,
newPackageForTaggingTemplate,
processedTaggingFilesTemplate,
+ errorDevTemplate,
bulkUploadTemplate,
manuscriptHasBeenPublishedTemplate,
}
diff --git a/server/ftp-integration/api.js b/server/ftp-integration/api.js
index 1c78f65ae..3e111967b 100644
--- a/server/ftp-integration/api.js
+++ b/server/ftp-integration/api.js
@@ -27,7 +27,7 @@ const config = require('config')
const rfr = require('rfr')
const xsweetConverter = rfr('server/xsweet-conversion')
-const { bulkUploaderEmail } = rfr('server/email')
+const { bulkUploaderEmail, errorDevEmail } = rfr('server/email')
// global.globalPath = ''
// global.globalTmpPath = ''
// global.manId = ''
@@ -42,6 +42,7 @@ const graphqlClient = new ApolloClient({
link,
})
const ftpUsers = config.get('users')
+const ftpDev = config.get('ftp_devteam')
const ignoreTaggerFolder = `${rootPath}${config.get('ftp_tagger').username}/*`
const watcher = chokidar.watch(`${rootPath}**/*.tar.gz`, {
@@ -91,6 +92,7 @@ process
if (err.submitter) {
bulkUploaderEmail(err.submitter.email, err.message)
}
+ errorDevEmail(ftpDev.email, err)
logger.error('whoops! there was an error', err.stack)
tidyUp(err.filePath, err.tmpPath, err.manId, true)
})
@@ -98,6 +100,7 @@ process
if (reason.submitter) {
bulkUploaderEmail(reason.submitter.email, reason.message)
}
+ errorDevEmail(ftpDev.email, reason)
logger.error(`Unhandled Rejection at: ${reason.stack || reason}`)
tidyUp(reason.filePath, reason.tmpPath, reason.manId, true)
})
diff --git a/server/ftp-integration/taggedXML.js b/server/ftp-integration/taggedXML.js
index 523ebf2e3..e4056aa6f 100644
--- a/server/ftp-integration/taggedXML.js
+++ b/server/ftp-integration/taggedXML.js
@@ -10,7 +10,14 @@ const config = require('config')
const manuscriptModel = require('../xpub-model/entities/manuscript/data-access')
const ftpTagger = config.get('ftp_tagger')
-const { processedTaggerEmail, bulkUploaderEmail } = require('../email')
+const ftpDev = config.get('ftp_devteam')
+const {
+ processedTaggerEmail,
+ bulkUploaderEmail,
+ errorDevEmail,
+} = require('../email')
+
+// const parentRootPath = '/home/yogmatee/projects/PLAY/chokidar/ftp_mock'
const parentRootPath = `${process.env.HOME}/${config.get('ftp_directory')}/${
ftpTagger.username
@@ -33,6 +40,22 @@ const watcher = chokidar.watch(`${rootPath}**/*.tar.gz`, {
},
})
+process
+ .on('uncaughtException', err => {
+ if (err.submitter) {
+ bulkUploaderEmail(err.submitter.email, err.message)
+ }
+ errorDevEmail(ftpDev.email, err)
+ logger.error('whoops! there was an error', err.stack)
+ })
+ .on('unhandledRejection', (reason, promise) => {
+ if (reason.submitter) {
+ bulkUploaderEmail(reason.submitter.email, reason.message)
+ }
+ errorDevEmail(ftpDev.email, reason)
+ logger.error(`Unhandled Rejection at: ${reason.stack || reason}`)
+ })
+
watcher
.on('add', path => logger.debug(`File ${path} has been added.`))
.on('add', path => processFile(path))
@@ -102,9 +125,10 @@ async function processFile(path) {
manuscriptObj[0]['meta,title'],
`${path}`,
)
+ // clean up tmp folder
+ await files.tidyUp(tmpPath)
} catch (err) {
logger.error('Error', err.message)
-
// send email to taggers
bulkUploaderEmail(ftpTagger.email, `${path}`)
}
diff --git a/server/ftp-integration/toTaggers.js b/server/ftp-integration/toTaggers.js
index cf5d2cd21..1a5511a33 100755
--- a/server/ftp-integration/toTaggers.js
+++ b/server/ftp-integration/toTaggers.js
@@ -10,9 +10,10 @@ const config = require('config')
const passport = require('passport')
const Manuscript = require('../xpub-model/entities/manuscript/data-access')
-const { taggerEmail } = rfr('server/email')
+const { taggerEmail, errorDevEmail } = rfr('server/email')
const pubsweetServer = config.get('pubsweet-server.baseUrl')
const ftpTagger = config.get('ftp_tagger')
+const ftpDev = config.get('ftp_devteam')
const ftpLocation = `${process.env.HOME}/${config.get('ftp_directory')}/${
ftpTagger.username
@@ -31,7 +32,7 @@ module.exports = app => {
})
}
-// createPackageForTaggers('EMS90002')
+// createPackageForTaggers('EMS92010')
// noinspection JSAnnotator
async function createPackageForTaggers(manid) {
@@ -64,7 +65,13 @@ function getManuscript(manId) {
manuscript.files = manuscript.files.filter(file => !file.deleted)
resolve(manuscript)
})
- .catch(err => reject(err))
+ .catch(err => {
+ errorDevEmail(
+ ftpDev.email,
+ `Issue with retrieving the manuscript with id ${manId}. Error: ${err}`,
+ )
+ reject(err)
+ })
})
}
@@ -80,7 +87,10 @@ function createTempDir(manuscript) {
tidyUp(directory)
}
fs.mkdir(directory, err => {
- if (err) reject(err)
+ if (err) {
+ errorDevEmail(ftpDev.email, err)
+ reject(err)
+ }
resolve(directory)
})
})
@@ -209,6 +219,7 @@ async function createMetadataXML(tmpPath, manuscript) {
return new Promise((resolve, reject) => {
fs.writeFile(`${tmpPath}/${manuscript.id}.xml`, xml, err => {
if (err) {
+ errorDevEmail(ftpDev.email, err)
reject(err)
}
resolve(`${manuscript.id}.xml`)
@@ -277,6 +288,7 @@ function createManifest(tmpPath, manuscript) {
return new Promise((resolve, reject) => {
fs.writeFile(`${tmpPath}/manifest.txt`, text, err => {
if (err) {
+ errorDevEmail(ftpDev.email, err)
reject(err)
}
resolve(true)
@@ -293,6 +305,7 @@ function compress(tmpPath, manuscript, datedFolder) {
exec(cmd, err => {
if (err) {
// node couldn't execute the command
+ errorDevEmail(ftpDev.email, err)
logger.error(err)
reject(err)
}
@@ -302,10 +315,17 @@ function compress(tmpPath, manuscript, datedFolder) {
})
}
-process.on('unhandledRejection', (reason, promise) => {
- logger.error(`Unhandled Rejection at: ${reason.stack || reason}`)
- tidyUp()
-})
+process
+ .on('uncaughtException', err => {
+ errorDevEmail(ftpDev.email, err)
+ logger.error('whoops! there was an error', err.stack)
+ // tidyUp()
+ })
+ .on('unhandledRejection', (reason, promise) => {
+ errorDevEmail(ftpDev.email, reason)
+ logger.error(`Unhandled Rejection at: ${reason.stack || reason}`)
+ // tidyUp()
+ })
function tidyUp(tmpPath) {
try {
diff --git a/server/utils/files.js b/server/utils/files.js
index 6f8a8dea9..863d5b728 100644
--- a/server/utils/files.js
+++ b/server/utils/files.js
@@ -6,6 +6,7 @@ const mime = require('mime-types')
const download = require('download')
const fetch = require('node-fetch')
const dateFormat = require('dateformat')
+const rimraf = require('rimraf')
const { minioClient } = require('express-middleware-minio')
@@ -141,6 +142,16 @@ module.exports.readData = function readData(url) {
})
}
+module.exports.tidyUp = function tidyUp(tmpPath) {
+ return new Promise((resolve, reject) => {
+ rimraf(tmpPath, err => {
+ if (err) reject(err)
+ logger.info(`successfully deleted ${tmpPath}`)
+ resolve(true)
+ })
+ })
+}
+
module.exports.fetchFile = function fetchFile(fileUrl) {
return new Promise((resolve, reject) => {
fetch(fileUrl, {
diff --git a/yarn.lock b/yarn.lock
index 6ea08f0d5..12568a880 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -12568,7 +12568,7 @@ right-align@^0.1.1:
dependencies:
align-text "^0.1.1"
-rimraf@2, rimraf@2.x.x, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@~2.6.2:
+rimraf@2, rimraf@2.x.x, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3, rimraf@~2.6.2:
version "2.6.3"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab"
integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==
--
GitLab