#1168 : Ensured that file renaming occurs on FTP, even for files that cannot be extracted

server/ftp-integration/api.js

@@ -75,8 +75,6 @@ const tidyUp = async (filePath, tmpPath, manId, isError) => {
         }
         if (isError) {
           logger.info(`Error in Bulk import process for ${filePath}`)
-          logger.info(`File has been moved to ${updatedFilepath}`)
-          // await updateFTP(updatedFilepath, submitter)
         } else {
           logger.info(
             `Finished Bulk import process for ${filePath} as manuscript ${manId}.`,
@@ -86,7 +84,7 @@ const tidyUp = async (filePath, tmpPath, manId, isError) => {
       })
     })
   } catch (e) {
-    logger.error('Error while tidying up after another error!')
+    logger.error(`An error has happened while processing file ${fileName}`)
   }
   return Journal.knex().destroy()
 }
@@ -181,7 +179,7 @@ const runProcess = async (packagePath, submitter) => {
     logger.info(`Starting Bulk Import Process for ${packagePath}`)
     packageName = path.basename(packagePath)
     tmpPath = await createTempDir()
-    const tmpPathExt = await extractFiles(packagePath, tmpPath)
+    const tmpPathExt = await extractFiles(packagePath, tmpPath, submitter)
     // rename file - append .start suffix
     suffixedFilePath = await fileUtils.renameFileSuffix(packagePath)
     const parsedInfo = await parseManifest(tmpPathExt, packageName)
@@ -222,15 +220,16 @@ const runProcess = async (packagePath, submitter) => {
   }
 }
 
-const extractFiles = (source, dest) =>
+const extractFiles = (source, dest, submitter) =>
   new Promise((resolve, reject) => {
     const cmd = `tar vzxf ${source} -C ${dest} --xform='s#^.+/##x'` // the xform removes all directories
     exec(cmd, async err => {
       if (err) {
-        logger.error('Error extracting FTP bulk upload files')
         // node couldn't execute the command
-        await tidyUp(source, dest, '', true)
-        logger.error(`${source} moved to Error folder`)
+        logger.error(`Error extracting FTP bulk upload file: ${source}`)
+        const updatedFilepath = await tidyUp(source, dest, '', true)
+        logger.info(`File ${source} has been moved to ${updatedFilepath}`)
+        await updateFTP(updatedFilepath, submitter)
         reject(err)
       }
       resolve(dest)
@@ -855,6 +854,6 @@ const updateFTP = async (updatedFilepath, submitter) => {
       processedFilename,
     )
   } catch (err) {
-    logger.error('Error with updating FTP')
+    logger.error('Error with updating FTP', err)
   }
 }

server/utils/ebi-ftp.js

@@ -134,8 +134,9 @@ module.exports.renameBulkFtpPackage = async function renameBulkFtpPackage(
             close(ftp)
             reject(err)
           }
-          // get array of tar.gz files only
           const ftpPackageName = await checkBulkFtpPackage(processedFilename)
+
+          // get array of tar.gz files only
           const ftpFile = list.filter(
             file => file.name === ftpPackageName && file.type === '-',
           )[0]
@@ -181,7 +182,7 @@ module.exports.renameBulkFtpPackage = async function renameBulkFtpPackage(
 function checkBulkFtpPackage(packageName) {
   return new Promise((resolve, reject) => {
     const file_ext = new RegExp(
-      /((\.start\.loaded\.mid-EMS\d+)|(\.start\.ERROR))\.\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}$/i,
+      /((\.start\.loaded\.mid-EMS\d+)|(\.start\.ERROR)|(\.ERROR))\.\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}$/i,
     )
     if (file_ext.test(packageName)) resolve(packageName.split(file_ext)[0])
     else