Unverified Commit cb7fb6fa authored by Jhoan Manuel Muñoz Serrano's avatar Jhoan Manuel Muñoz Serrano Committed by GitHub

Pivotal ID # 172101894: Remove old studies_partial_* files (#139)

Create a task to delete the unnecessary partial updates files
parent e6b59942
Pipeline #75638 passed with stages
in 3 minutes and 58 seconds
package uk.ac.ebi.biostd.exporter.jobs.partial;
import java.io.File;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import uk.ac.ebi.biostd.exporter.utils.FileUtil;
@Slf4j
@Component
@AllArgsConstructor
public class PartialExportCleaner {
private final PartialExportJobProperties configProperties;
public void execute() {
log.info("cleaning up partial export files");
FileUtil
.listFilesMatching(configProperties.getFilePath(), configProperties.getFileName() + ".+")
.forEach(File::delete);
}
}
......@@ -9,6 +9,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.config.CronTask;
import org.springframework.stereotype.Component;
import uk.ac.ebi.biostd.exporter.jobs.common.api.ExportPipeline;
import uk.ac.ebi.biostd.exporter.jobs.partial.PartialExportCleaner;
import uk.ac.ebi.biostd.exporter.jobs.partial.PartialSubmissionExporter;
import uk.ac.ebi.biostd.exporter.jobs.releaser.ReleaserJob;
......@@ -26,6 +27,7 @@ public class ScheduledTasksConfiguration {
private final ExportPipeline pmcExporter;
private final ExportPipeline statsExporter;
private final PartialSubmissionExporter partialExporter;
private final PartialExportCleaner partialExportCleaner;
private final ReleaserJob releaserJob;
@Value("${jobs.dummy.cron}")
......@@ -37,6 +39,9 @@ public class ScheduledTasksConfiguration {
@Value("${jobs.partial.cron:''}")
private String partialCron;
@Value("${jobs.partial.cleanerCron:''}")
private String partialCleanerCron;
@Value("${jobs.pmc.export.cron:''}")
private String pmcCron;
......@@ -51,9 +56,11 @@ public class ScheduledTasksConfiguration {
@Qualifier("pmc") ExportPipeline pmcExporter,
@Qualifier("stats") ExportPipeline statsExporter,
PartialSubmissionExporter partialExporter,
PartialExportCleaner partialExportCleaner,
ReleaserJob releaserJob) {
this.fullExporter = fullExporter;
this.partialExporter = partialExporter;
this.partialExportCleaner = partialExportCleaner;
this.pmcExporter = pmcExporter;
this.statsExporter = statsExporter;
this.releaserJob = releaserJob;
......@@ -77,6 +84,12 @@ public class ScheduledTasksConfiguration {
return new CronTask(new PartialExportJob(partialExporter::execute), partialCron);
}
@Bean
@ConditionalOnProperty(prefix = "jobs.partial", name = "enabled", havingValue = "true")
public CronTask partialCleanerScheduler() {
return new CronTask(new PartialExportCleanerJob(partialExportCleaner::execute), partialCleanerCron);
}
@Bean
@ConditionalOnProperty(prefix = "jobs.pmc.export", name = "enabled", havingValue = "true")
public CronTask pmcScheduler() {
......@@ -117,6 +130,17 @@ public class ScheduledTasksConfiguration {
}
}
@AllArgsConstructor
public static class PartialExportCleanerJob implements Runnable {
private final Runnable runnable;
@Override
public void run() {
runnable.run();
}
}
@AllArgsConstructor
public static class PmcLinksExportJob implements Runnable {
......
......@@ -24,19 +24,12 @@ public class FileUtil {
return Files.list(Paths.get(filePath)).map(Path::toFile).collect(toList());
}
public static String filesAsStringList(List<File> files) {
StringBuilder builder = new StringBuilder();
files.forEach(file -> {
builder.append("- ");
builder.append(file.toPath());
builder.append("\n");
});
return builder.toString();
}
public static File[] listFilesMatching(String rootPath, String regex) {
Pattern p = Pattern.compile(regex);
return new File(rootPath).listFiles(file -> p.matcher(file.getName()).matches());
@SneakyThrows
public static List<File> listFilesMatching(String rootPath, String regex) {
Pattern pattern = Pattern.compile(regex);
return Files.list(Paths.get(rootPath))
.map(Path::toFile)
.filter(file -> pattern.matcher(file.getName()).matches())
.collect(toList());
}
}
......@@ -54,6 +54,7 @@ jobs:
notificationUrl: http://localhost:8586/api/update/partial/
deleteNotificationUrl: http://localhost:8586/api/delete/
cron: 0/20 * * * * ? # each 20 seconds
cleanerCron: 0 0 6 * * ? # everyday at 6 a.m.
full:
enabled: false
......@@ -102,6 +103,7 @@ jobs:
notificationUrl: http://ribs:8080/biostudies/api/v1/index/reload/
deleteNotificationUrl: http://ribs:8080/biostudies/api/v1/index/delete/
cron: 0 0/5 5-19 * * ? # Every 5 minutes, between 05:00 AM and 07:59 PM.
cleanerCron: 0 3 1 * * ? # everyday at 1 a.m.
full:
enabled: true
allSubmissions:
......@@ -159,6 +161,7 @@ jobs:
notificationUrl: http://ves-ebi-6b:8080/biostudies/api/v1/index/reload/
deleteNotificationUrl: http://ves-ebi-6b:8080/biostudies/api/v1/index/delete/
cron: 0 0/5 5-19 * * ? # each five minutes between 5 am and 7 pm.
cleanerCron: 0 3 2 * * ? # everyday at 2 a.m.
full:
enabled: true
allSubmissions:
......@@ -205,6 +208,7 @@ jobs:
fileName: studies_partial
notificationUrl: http://ves-ebi-6e:8080/biostudies/preview/api/v1/index/reload/
cron: 0 0/5 5-19 * * ? # each five minutes between 5 am and 7 pm.
cleanerCron: 0 3 3 * * ? # everyday at 3 a.m.
full:
enabled: true
allSubmissions:
......
package uk.ac.ebi.biostd.exporter.jobs.partial;
import static org.junit.Assert.assertFalse;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
public class PartialExporterCleanerTest {
private String tempFolderRoot;
private PartialExportCleaner testInstance;
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
@Before
public void setUp() throws IOException {
tempFolderRoot = tempFolder.getRoot().getAbsolutePath();
setUpTestFiles();
testInstance = new PartialExportCleaner(createTestProperties());
}
@Test
public void execute() {
testInstance.execute();
assertFalse(Files.exists(Paths.get(tempFolderRoot + "studies_partial_2020_04_05_07_35.json")));
assertFalse(Files.exists(Paths.get(tempFolderRoot + "studies_partial_2020_04_05_13_25.json")));
}
private void setUpTestFiles() throws IOException {
tempFolder.newFile("studies_partial_2020_04_05_07_35.json");
tempFolder.newFile("studies_partial_2020_04_05_13_25.json");
}
private PartialExportJobProperties createTestProperties() {
PartialExportJobProperties testConfig = new PartialExportJobProperties();
testConfig.setFileName("studies_partial");
testConfig.setFilePath(tempFolderRoot);
return testConfig;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment