Commit f2bee407 authored by M. Haseeb's avatar M. Haseeb

ENA-3431

- added a new flag that lets SAM and Fastq validation complete quickly.
parent 150ad7fc
......@@ -11,14 +11,14 @@ plugins {
apply plugin: 'io.spring.dependency-management'
group = 'uk.ac.ebi.ena.webin-cli'
version = '3.4.0'
version = '3.5.0'
sourceCompatibility = '1.8'
targetCompatibility = '1.8'
spotless {
java {
licenseHeader('''/*
* Copyright 2018-2019 EMBL - European Bioinformatics Institute
* Copyright 2018-2021 EMBL - European Bioinformatics Institute
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
* file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
......@@ -66,7 +66,7 @@ repositories {
maven { url "https://gitlab.ebi.ac.uk/api/v4/groups/enasequence/-/packages/maven" }
mavenCentral()
}
//
task printWebinTestCredentials {
doLast {
println 'webinCliUsername: ' + webinCliUsername
......@@ -75,10 +75,10 @@ task printWebinTestCredentials {
}
dependencies {
implementation( 'uk.ac.ebi.ena.webin-cli:webin-cli-validator:1.1.0' )
implementation( 'uk.ac.ebi.ena.webin-cli:webin-cli-validator:1.2.0' )
implementation( 'uk.ac.ebi.ena.sequence:sequencetools:2.0.86' )
implementation( 'uk.ac.ebi.ena.txmbtools:txmbtools:1.0.3' )
implementation( 'uk.ac.ebi.ena:readtools:1.9.0' )
implementation( 'uk.ac.ebi.ena:readtools:1.11.0' )
implementation( 'org.springframework.boot:spring-boot-starter' )
implementation( 'org.springframework.boot:spring-boot-starter-web' ) {
exclude group: 'org.springframework.boot', module: 'spring-boot-starter-tomcat'
......
......@@ -10,23 +10,6 @@
*/
package uk.ac.ebi.ena.webin.cli;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.fusesource.jansi.AnsiConsole;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
import ch.qos.logback.classic.spi.ILoggingEvent;
......@@ -36,10 +19,19 @@ import de.vandermeer.asciitable.AT_Renderer;
import de.vandermeer.asciitable.AsciiTable;
import de.vandermeer.asciitable.CWC_FixedWidth;
import de.vandermeer.skb.interfaces.transformers.textformat.TextAlignment;
import org.apache.commons.lang3.StringUtils;
import org.fusesource.jansi.AnsiConsole;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine;
import uk.ac.ebi.ena.webin.cli.entity.Version;
import uk.ac.ebi.ena.webin.cli.manifest.*;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldDefinition;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldType;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileCount;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileGroup;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestReader;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestReaderBuilder;
import uk.ac.ebi.ena.webin.cli.manifest.processor.CVFieldProcessor;
import uk.ac.ebi.ena.webin.cli.service.LoginService;
import uk.ac.ebi.ena.webin.cli.service.SubmitService;
......@@ -49,6 +41,22 @@ import uk.ac.ebi.ena.webin.cli.upload.ASCPService;
import uk.ac.ebi.ena.webin.cli.upload.FtpService;
import uk.ac.ebi.ena.webin.cli.upload.UploadService;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
public class WebinCli {
public final static int SUCCESS = 0;
public final static int SYSTEM_ERROR = 1;
......@@ -128,6 +136,7 @@ public class WebinCli {
parameters.setPassword(cmd.password);
parameters.setCenterName(cmd.centerName);
parameters.setValidate(cmd.validate);
parameters.setQuick(cmd.quick);
parameters.setSubmit(cmd.submit);
parameters.setTest(cmd.test);
parameters.setAscp(cmd.ascp);
......
......@@ -10,11 +10,11 @@
*/
package uk.ac.ebi.ena.webin.cli;
import java.io.File;
import picocli.CommandLine;
import picocli.CommandLine.Option;
import java.io.File;
@CommandLine.Command(
headerHeading = "%n",
descriptionHeading = "%nDescription:%n%n",
......@@ -62,16 +62,19 @@ WebinCliCommand
@Option(names = { Options.centerName, Options.centerNameSynonym }, description = Descriptions.centerName, paramLabel= "CENTER", order = 6)
public String centerName;
@Option(names = Options.validate, description = Descriptions.validate, order = 7)
@Option(names = Options.validate, description = Descriptions.validate, defaultValue = "false", order = 7)
public boolean validate;
@Option(names = Options.submit, description = Descriptions.submit, order = 8)
@Option(names = Options.quick, description = Descriptions.quick, order = 8)
public boolean quick;
@Option(names = Options.submit, description = Descriptions.submit, order = 9)
public boolean submit;
@Option(names = Options.test, description = Descriptions.test, order = 9)
@Option(names = Options.test, description = Descriptions.test, order = 10)
public boolean test;
@Option(names = Options.ascp, description = Descriptions.ascp, order = 10)
@Option(names = Options.ascp, description = Descriptions.ascp, order = 11)
public boolean ascp;
@Option(names = Options.help, description = Descriptions.help, usageHelp = true, order = 100)
......@@ -95,6 +98,7 @@ WebinCliCommand
String centerName = "-centerName";
String submit = "-submit";
String validate = "-validate";
String quick = "-quick";
String test = "-test";
String ascp = "-ascp";
String help = "-help";
......@@ -126,6 +130,7 @@ WebinCliCommand
String centerName = "Mandatory center name for broker accounts.";
String submit = "Validate, upload and submit files.";
String validate = "Validate files without uploading or submitting them.";
String quick = "Validates submitted read files (BAM, CRAM, Fastq) within a fixed time period (5 minutes). All CRAM reference sequence md5 checksums are always validated. When this option is used files may only be partially validated and may fail post-submission processing.";
String test = "Use the test submission service.";
String ascp =
"Use Aspera (if Aspera Cli is available) instead of FTP when uploading files. " +
......
......@@ -10,10 +10,14 @@
*/
package uk.ac.ebi.ena.webin.cli;
import java.io.File;
import uk.ac.ebi.ena.webin.cli.manifest.processor.MetadataProcessorParameters;
import uk.ac.ebi.ena.webin.cli.manifest.processor.metadata.*;
import uk.ac.ebi.ena.webin.cli.manifest.processor.metadata.AnalysisProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.metadata.RunProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.metadata.SampleProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.metadata.SampleXmlProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.metadata.StudyProcessor;
import java.io.File;
public class WebinCliParameters implements MetadataProcessorParameters {
private String submissionAccount;
......@@ -25,6 +29,7 @@ public class WebinCliParameters implements MetadataProcessorParameters {
private File inputDir = new File(".");
private String centerName;
private boolean validate;
private boolean quick;
private boolean submit;
private boolean test;
private boolean ascp;
......@@ -113,6 +118,14 @@ public class WebinCliParameters implements MetadataProcessorParameters {
this.validate = validate;
}
public boolean isQuick() {
return quick;
}
public void setQuick(boolean quick) {
this.quick = quick;
}
public boolean isSubmit() {
return submit;
}
......
......@@ -10,16 +10,28 @@
*/
package uk.ac.ebi.ena.webin.cli.context.genome;
import java.util.ArrayList;
import java.util.Map;
import uk.ac.ebi.ena.webin.cli.WebinCliMessage;
import uk.ac.ebi.ena.webin.cli.manifest.*;
import uk.ac.ebi.ena.webin.cli.manifest.processor.*;
import uk.ac.ebi.ena.webin.cli.WebinCliParameters;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestCVList;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldDefinition;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileCount;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileGroup;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileSuffix;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestReader;
import uk.ac.ebi.ena.webin.cli.manifest.processor.ASCIIFileNameProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.AssemblyTypeProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.AuthorProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.CVFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.FileSuffixProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.MetadataProcessorFactory;
import uk.ac.ebi.ena.webin.cli.validator.file.SubmissionFile;
import uk.ac.ebi.ena.webin.cli.validator.file.SubmissionFiles;
import uk.ac.ebi.ena.webin.cli.validator.manifest.GenomeManifest;
import java.util.ArrayList;
import java.util.Map;
public class
GenomeManifestReader extends ManifestReader<GenomeManifest> {
......@@ -104,7 +116,7 @@ GenomeManifestReader extends ManifestReader<GenomeManifest> {
private final GenomeManifest manifest = new GenomeManifest();
public GenomeManifestReader(
ManifestReaderParameters parameters,
WebinCliParameters parameters,
MetadataProcessorFactory factory)
{
super( parameters,
......@@ -178,6 +190,10 @@ GenomeManifestReader extends ManifestReader<GenomeManifest> {
manifest.getSample().setAttributes(sample.getAttributes());
});
}
if (parameters != null) {
manifest.setQuick(parameters.isQuick());
}
}
private static ManifestFieldProcessor[] getChromosomeListProcessors() {
......
......@@ -10,19 +10,27 @@
*/
package uk.ac.ebi.ena.webin.cli.context.reads;
import java.io.File;
import java.util.stream.Stream;
import org.apache.commons.lang.StringUtils;
import uk.ac.ebi.ena.webin.cli.WebinCliMessage;
import uk.ac.ebi.ena.webin.cli.manifest.*;
import uk.ac.ebi.ena.webin.cli.manifest.processor.*;
import uk.ac.ebi.ena.webin.cli.WebinCliParameters;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestCVList;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldDefinition;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileCount;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileSuffix;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestReader;
import uk.ac.ebi.ena.webin.cli.manifest.processor.ASCIIFileNameProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.CVFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.FileSuffixProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.MetadataProcessorFactory;
import uk.ac.ebi.ena.webin.cli.validator.file.SubmissionFile;
import uk.ac.ebi.ena.webin.cli.validator.file.SubmissionFiles;
import uk.ac.ebi.ena.webin.cli.validator.manifest.ReadsManifest;
import uk.ac.ebi.ena.webin.cli.validator.manifest.ReadsManifest.QualityScore;
import java.io.File;
import java.util.stream.Stream;
public class
ReadsManifestReader extends ManifestReader<ReadsManifest> {
......@@ -86,7 +94,7 @@ ReadsManifestReader extends ManifestReader<ReadsManifest> {
public ReadsManifestReader(
ManifestReaderParameters parameters,
WebinCliParameters parameters,
MetadataProcessorFactory factory)
{
super(parameters,
......@@ -132,6 +140,10 @@ ReadsManifestReader extends ManifestReader<ReadsManifest> {
if ( factory.getSampleProcessor() != null ) {
factory.getSampleProcessor().setCallback(sample -> manifest.setSample(sample));
}
if (parameters != null) {
manifest.setQuick(parameters.isQuick());
}
}
private static ManifestFieldProcessor[] getFastqProcessors() {
......
......@@ -10,15 +10,23 @@
*/
package uk.ac.ebi.ena.webin.cli.context.sequence;
import java.util.Map;
import uk.ac.ebi.ena.webin.cli.WebinCliMessage;
import uk.ac.ebi.ena.webin.cli.manifest.*;
import uk.ac.ebi.ena.webin.cli.manifest.processor.*;
import uk.ac.ebi.ena.webin.cli.WebinCliParameters;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldDefinition;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileCount;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileSuffix;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestReader;
import uk.ac.ebi.ena.webin.cli.manifest.processor.ASCIIFileNameProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.AuthorProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.FileSuffixProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.MetadataProcessorFactory;
import uk.ac.ebi.ena.webin.cli.validator.file.SubmissionFile;
import uk.ac.ebi.ena.webin.cli.validator.file.SubmissionFiles;
import uk.ac.ebi.ena.webin.cli.validator.manifest.SequenceManifest;
import java.util.Map;
public class
SequenceManifestReader extends ManifestReader<SequenceManifest>
{
......@@ -54,7 +62,7 @@ SequenceManifestReader extends ManifestReader<SequenceManifest>
private final SequenceManifest manifest = new SequenceManifest();
public SequenceManifestReader(
ManifestReaderParameters parameters,
WebinCliParameters parameters,
MetadataProcessorFactory factory) {
super(parameters,
// Fields.
......@@ -91,6 +99,10 @@ SequenceManifestReader extends ManifestReader<SequenceManifest>
if (factory.getAnalysisProcessor() != null ) {
factory.getAnalysisProcessor().setCallback(analysis -> manifest.setAnalysis(analysis));
}
if (parameters != null) {
manifest.setQuick(parameters.isQuick());
}
}
private static ManifestFieldProcessor[] getTabProcessors() {
......
......@@ -10,7 +10,12 @@
*/
package uk.ac.ebi.ena.webin.cli.context.taxrefset;
import uk.ac.ebi.ena.webin.cli.manifest.*;
import uk.ac.ebi.ena.webin.cli.WebinCliParameters;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldDefinition;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileCount;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileSuffix;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestReader;
import uk.ac.ebi.ena.webin.cli.manifest.processor.ASCIIFileNameProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.CustomFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.FileSuffixProcessor;
......@@ -53,7 +58,7 @@ public class TaxRefSetManifestReader extends ManifestReader<TaxRefSetManifest> {
}
public TaxRefSetManifestReader(ManifestReaderParameters parameters,
public TaxRefSetManifestReader(WebinCliParameters parameters,
MetadataProcessorFactory factory) {
super( parameters,
// Fields.
......@@ -82,6 +87,7 @@ public class TaxRefSetManifestReader extends ManifestReader<TaxRefSetManifest> {
getCustomFieldProcessor().setCallback(keyVal-> manifest.addCustomField(keyVal.left,keyVal.right));
manifest.setQuick(parameters.isQuick());
}
private static CustomFieldProcessor getCustomFieldProcessor() {
......
......@@ -10,15 +10,25 @@
*/
package uk.ac.ebi.ena.webin.cli.context.transcriptome;
import java.util.Map;
import uk.ac.ebi.ena.webin.cli.WebinCliMessage;
import uk.ac.ebi.ena.webin.cli.manifest.*;
import uk.ac.ebi.ena.webin.cli.manifest.processor.*;
import uk.ac.ebi.ena.webin.cli.WebinCliParameters;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestCVList;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldDefinition;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileCount;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestFileSuffix;
import uk.ac.ebi.ena.webin.cli.manifest.ManifestReader;
import uk.ac.ebi.ena.webin.cli.manifest.processor.ASCIIFileNameProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.AuthorProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.CVFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.FileSuffixProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.MetadataProcessorFactory;
import uk.ac.ebi.ena.webin.cli.validator.file.SubmissionFile;
import uk.ac.ebi.ena.webin.cli.validator.file.SubmissionFiles;
import uk.ac.ebi.ena.webin.cli.validator.manifest.TranscriptomeManifest;
import java.util.Map;
public class
TranscriptomeManifestReader extends ManifestReader<TranscriptomeManifest>
{
......@@ -71,7 +81,7 @@ TranscriptomeManifestReader extends ManifestReader<TranscriptomeManifest>
private final TranscriptomeManifest manifest = new TranscriptomeManifest();
public TranscriptomeManifestReader(
ManifestReaderParameters parameters,
WebinCliParameters parameters,
MetadataProcessorFactory factory)
{
super(parameters,
......@@ -124,6 +134,9 @@ TranscriptomeManifestReader extends ManifestReader<TranscriptomeManifest>
});
}
if (parameters != null) {
manifest.setQuick(parameters.isQuick());
}
}
private static ManifestFieldProcessor[] getFastaProcessors() {
......
......@@ -10,29 +10,38 @@
*/
package uk.ac.ebi.ena.webin.cli.manifest;
import static uk.ac.ebi.ena.webin.cli.manifest.ManifestReader.Fields.INFO;
import static uk.ac.ebi.ena.webin.cli.manifest.ManifestReader.ManifestReaderState.State.PARSE;
import static uk.ac.ebi.ena.webin.cli.manifest.ManifestReader.ManifestReaderState.State.VALIDATE;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.lang.StringUtils;
import uk.ac.ebi.ena.webin.cli.WebinCliMessage;
import uk.ac.ebi.ena.webin.cli.WebinCliParameters;
import uk.ac.ebi.ena.webin.cli.validator.manifest.Manifest;
import uk.ac.ebi.ena.webin.cli.validator.message.ValidationMessage;
import uk.ac.ebi.ena.webin.cli.validator.message.ValidationOrigin;
import uk.ac.ebi.ena.webin.cli.validator.message.ValidationResult;
import uk.ac.ebi.ena.webin.cli.validator.message.listener.MessageListener;
import java.io.*;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.lang.StringUtils;
import uk.ac.ebi.ena.webin.cli.WebinCliMessage;
import uk.ac.ebi.ena.webin.cli.validator.manifest.Manifest;
import uk.ac.ebi.ena.webin.cli.validator.message.ValidationMessage;
import uk.ac.ebi.ena.webin.cli.validator.message.ValidationOrigin;
import uk.ac.ebi.ena.webin.cli.validator.message.ValidationResult;
import uk.ac.ebi.ena.webin.cli.validator.message.listener.MessageListener;
import static uk.ac.ebi.ena.webin.cli.manifest.ManifestReader.Fields.INFO;
import static uk.ac.ebi.ena.webin.cli.manifest.ManifestReader.ManifestReaderState.State.PARSE;
import static uk.ac.ebi.ena.webin.cli.manifest.ManifestReader.ManifestReaderState.State.VALIDATE;
public abstract class
ManifestReader<M extends Manifest> {
......@@ -76,41 +85,28 @@ ManifestReader<M extends Manifest> {
int lineNo = 0;
}
private final ManifestReaderParameters parameters;
private final WebinCliParameters webinCliParameters;
private final List<ManifestFieldDefinition> fields;
private final List<ManifestFileGroup> fileGroups;
private List<MessageListener> listener = new ArrayList<>();
private ManifestReaderResult manifestReaderResult;
private ManifestReaderState state;
public static final ManifestReaderParameters DEFAULT_PARAMETERS = new ManifestReaderParameters() {
public boolean isManifestValidateMandatory() {
return true;
}
public boolean isManifestValidateFileExist() {
return true;
}
public boolean isManifestValidateFileCount() {
return true;
}
};
public
ManifestReader( ManifestReaderParameters parameters,
ManifestReader( WebinCliParameters webinCliParameters,
List<ManifestFieldDefinition> fields )
{
this.parameters = parameters;
this.webinCliParameters = webinCliParameters;
this.fields = fields;
this.fileGroups = null;
}
public
ManifestReader( ManifestReaderParameters parameters,
ManifestReader( WebinCliParameters webinCliParameters,
List<ManifestFieldDefinition> fields,
List<ManifestFileGroup> fileGroups)
{
this.parameters = parameters;
this.webinCliParameters = webinCliParameters;
this.fields = fields;
this.fileGroups = fileGroups;
}
......@@ -287,9 +283,7 @@ ManifestReader<M extends Manifest> {
if( field.getDefinition().getType() == ManifestFieldType.FILE )
{
// Validate file exists.
if (parameters.isManifestValidateFileExist()) {
validateFileExists( inputDir, field );
}
validateFileExists( inputDir, field );
}
return field;
......@@ -312,20 +306,17 @@ ManifestReader<M extends Manifest> {
// Validate min count.
if (parameters.isManifestValidateMandatory()) {
fields.stream()
.filter( field -> field.getMinCount() > 0 )
.forEach( minCountField -> {
if( manifestReaderResult.getFields()
.stream()
.filter( field -> field.getName().equals( minCountField.getName() ) )
.count() < 1 )
{
error( WebinCliMessage.MANIFEST_READER_MISSING_MANDATORY_FIELD_ERROR, minCountField.getName() );
}
} );
}
fields.stream()
.filter( field -> field.getMinCount() > 0 )
.forEach( minCountField -> {
if( manifestReaderResult.getFields()
.stream()
.filter( field -> field.getName().equals( minCountField.getName() ) )
.count() < 1 )
{
error( WebinCliMessage.MANIFEST_READER_MISSING_MANDATORY_FIELD_ERROR, minCountField.getName() );
}
} );
// Validate max count.
......@@ -406,50 +397,47 @@ ManifestReader<M extends Manifest> {
if( fileGroups == null || fileGroups.isEmpty() )
return;
if (parameters.isManifestValidateFileCount()) {
Map<String, Long> fileCountMap = manifestReaderResult.getFields()
.stream()
.filter( field -> field.getDefinition().getType().equals( ManifestFieldType.FILE ) )
.collect( Collectors.groupingBy( ManifestFieldValue::getName, Collectors.counting() ) );
Map<String, Long> fileCountMap = manifestReaderResult.getFields()
.stream()
.filter( field -> field.getDefinition().getType().equals( ManifestFieldType.FILE ) )
.collect( Collectors.groupingBy( ManifestFieldValue::getName, Collectors.counting() ) );
if( fileCountMap == null || fileCountMap.isEmpty() )
{
error(WebinCliMessage.MANIFEST_READER_NO_DATA_FILES_ERROR, getFileGroupText(fileGroups));
return;
}
if( fileCountMap == null || fileCountMap.isEmpty() )
{
error(WebinCliMessage.MANIFEST_READER_NO_DATA_FILES_ERROR, getFileGroupText(fileGroups));
return;
}
next:
for (ManifestFileGroup fileGroup : fileGroups) {
for (ManifestFileCount fileCount : fileGroup.getFileCounts()) {
if (fileCountMap.get(fileCount.getFileType()) == null) {
if (fileCount.getMinCount() > 0) {