Commit 95863bf3 authored by M. Haseeb's avatar M. Haseeb

ENA-4161

- review followup.
- added support for manifest file key punctuations.
parent 3589a60f
......@@ -453,8 +453,6 @@ public class WebinCli {
StringBuilder attHelpText = new StringBuilder();
if (!field.getFieldAttributes().isEmpty()) {
//attHelpText.append("<br/><br/>Attributes:");
field.getFieldAttributes().stream().forEach(att -> {
attHelpText.append("<br/>" + att.getName() + " attribute");
......
......@@ -120,19 +120,14 @@ ReadsManifestReader extends ManifestReader<ReadsManifest> {
.meta().optional().name(Field.LIBRARY_NAME).desc(Description.LIBRARY_NAME).and()
.meta().optional().name(Field.INSERT_SIZE).desc(Description.INSERT_SIZE).and()
.file()
.optional(2)
.name(Field.FASTQ)
.desc(Description.FASTQ)
.processor(getFastqProcessors())
.attributes(new ManifestFieldDefinition.Builder()
.attribute().optional().name(Field.READ_TYPE).desc(Description.READ_TYPE)
.processor(new CVFieldProcessor(CV_READ_TYPE)).build())
.and()
.file().optional(2).name(Field.FASTQ).desc(Description.FASTQ).processor(getFastqProcessors())
.attributes(new ManifestFieldDefinition.Builder().attribute().optional().name(Field.READ_TYPE)
.desc(Description.READ_TYPE).processor(new CVFieldProcessor(CV_READ_TYPE)).build())
.and()
.file().optional().name(Field.BAM).desc(Description.BAM).processor(getBamProcessors()).and()
.file().optional().name(Field.CRAM).desc(Description.CRAM).processor(getCramProcessors()).and()
.meta().optional().hidden().name(Field.QUALITY_SCORE).desc(Description.QUALITY_SCORE).processor(new CVFieldProcessor(CV_QUALITY_SCORE)).and()
.meta().optional().hidden().name(Field.QUALITY_SCORE).desc(Description.QUALITY_SCORE).processor(new CVFieldProcessor(CV_QUALITY_SCORE)).and()
.meta().optional().hidden().name(Field.__HORIZON).desc(Description.__HORIZON).and()
.meta().optional().name(Fields.SUBMISSION_TOOL).desc(Descriptions.SUBMISSION_TOOL).and()
.meta().optional().name(Fields.SUBMISSION_TOOL_VERSION).desc(Descriptions.SUBMISSION_TOOL_VERSION)
......
......@@ -49,7 +49,7 @@ import static uk.ac.ebi.ena.webin.cli.manifest.ManifestReader.ManifestReaderStat
public abstract class
ManifestReader<M extends Manifest> {
public static final String FIELD_NAME_REGEX = "^[\\s]*(#|;|\\/\\/).*$";
public static final String KEY_VALUE_COMMENT_REGEX = "^[\\s]*(#|;|\\/\\/).*$";
public abstract M getManifest();
......@@ -234,9 +234,9 @@ ManifestReader<M extends Manifest> {
parseManifest( Path inputDir, List<String> lines )
{
if (isJsonBasedFormat(inputDir, lines)) {
parseManifestNew(inputDir, lines);
parseJsonManifest(inputDir, lines);
} else {
parseManifestOld(inputDir, lines);
parseKeyValueManifest(inputDir, lines);
}
}
......@@ -249,7 +249,7 @@ ManifestReader<M extends Manifest> {
}
private void parseManifestOld( Path inputDir, List<String> lines )
private void parseKeyValueManifest(Path inputDir, List<String> lines )
{
state.state = PARSE;
......@@ -281,7 +281,7 @@ ManifestReader<M extends Manifest> {
String fieldName = StringUtils.stripEnd( tokens[ 0 ].trim().toUpperCase(), ": " );
String fieldValue = ( tokens.length == 2 ) ? tokens[ 1 ].trim() : null;
if( fieldName.matches( FIELD_NAME_REGEX ) )
if( fieldName.matches(KEY_VALUE_COMMENT_REGEX) ) // Ignore comment lines.
return null;
try
......@@ -289,7 +289,7 @@ ManifestReader<M extends Manifest> {
ManifestFieldDefinition fieldDefinition = Stream
.concat( infoFields.stream(), fields.stream() )
.filter(
field -> field.getName().equalsIgnoreCase( fieldName ) ||
field -> matchNameCaseAndPunctuationInsensitively(field.getName(), fieldName) ||
field.matchSynonym( fieldName ))
.findFirst()
.get();
......@@ -321,7 +321,7 @@ ManifestReader<M extends Manifest> {
return null;
}
private void parseManifestNew( Path inputDir, List<String> lines ) {
private void parseJsonManifest(Path inputDir, List<String> lines ) {
state.state = PARSE;
try {
......@@ -332,12 +332,9 @@ ManifestReader<M extends Manifest> {
jsonNode.fields().forEachRemaining(field -> {
String fieldName = field.getKey();
if( fieldName.matches( FIELD_NAME_REGEX ) )
return;
//find field definition
ManifestFieldDefinition fieldDefinition = Stream.concat( infoFields.stream(), fields.stream() )
.filter(fieldDef -> fieldDef.getName().equalsIgnoreCase( fieldName ) ||
.filter(fieldDef -> matchNameCaseAndPunctuationInsensitively(fieldDef.getName(), fieldName) ||
fieldDef.matchSynonym( fieldName ))
.findFirst().orElse(null);
if (fieldDefinition == null) {
......@@ -358,12 +355,10 @@ ManifestReader<M extends Manifest> {
if (fieldData.has("attributes") && !fieldDefinition.getFieldAttributes().isEmpty()) {
fieldData.get("attributes").fields().forEachRemaining(att -> {
String attName = att.getKey();
if( attName.matches( FIELD_NAME_REGEX ) )
return;
//find attribute definition in field's attributes definitions.
ManifestFieldDefinition attDef = fieldDefinition.getFieldAttributes().stream()
.filter(attFieldDef -> attFieldDef.getName().equalsIgnoreCase( attName ) ||
.filter(attFieldDef -> matchNameCaseAndPunctuationInsensitively(attFieldDef.getName(), attName) ||
attFieldDef.matchSynonym( attName ))
.findFirst().orElse(null);
if (attDef == null) {
......@@ -778,4 +773,9 @@ ManifestReader<M extends Manifest> {
public WebinCliParameters getWebinCliParameters() {
return webinCliParameters;
}
private boolean matchNameCaseAndPunctuationInsensitively(String name1, String name2) {
return name1.replaceAll("[ _-]+", "")
.equalsIgnoreCase(name2.replaceAll("[ _-]+", ""));
}
}
......@@ -21,14 +21,19 @@ import java.nio.file.Path;
public class ManifestBuilder {
private boolean isNew;
public enum ManifestFormat {
KEY_VALUE,
JSON
}
private ManifestFormat manifestFormat = ManifestFormat.KEY_VALUE;
private String manifest = "";
private ObjectNode jsonManifest;
public ManifestBuilder newFormat() {
isNew = true;
public ManifestBuilder jsonFormat() {
manifestFormat = ManifestFormat.JSON;
return this;
}
......@@ -51,7 +56,7 @@ public class ManifestBuilder {
}
public ManifestBuilder field(String field, String value) {
if (!isNew) {
if (manifestFormat == ManifestFormat.KEY_VALUE) {
if (field != null && value != null) {
manifest += field + "\t" + value + "\n";
}
......@@ -67,7 +72,7 @@ public class ManifestBuilder {
}
public ManifestBuilder attribute(String field, String attributeKey, String attributeValue) {
if (!isNew) {
if (manifestFormat != ManifestFormat.JSON) {
return this;
}
......@@ -182,7 +187,7 @@ public class ManifestBuilder {
}
public File build() {
if (!isNew) {
if (manifestFormat == ManifestFormat.KEY_VALUE) {
return TempFileBuilder.file(manifest).toFile();
} else {
try {
......
......@@ -305,7 +305,7 @@ ReadsManifestReaderTest {
ReadsManifest manifest = manifestReader.getManifest();
manifestReader.readManifest(Paths.get("."),
new ManifestBuilder().newFormat()
new ManifestBuilder().jsonFormat()
.field(Field.PLATFORM, "illumina")
.field(Field.INSTRUMENT, "Illumina HiScanSQ")
.field(Field.LIBRARY_STRATEGY, "CLONEEND")
......
......@@ -17,10 +17,12 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import uk.ac.ebi.ena.webin.cli.ManifestBuilder;
import uk.ac.ebi.ena.webin.cli.WebinCliMessage;
import uk.ac.ebi.ena.webin.cli.manifest.processor.CVFieldProcessor;
import uk.ac.ebi.ena.webin.cli.manifest.processor.FileSuffixProcessor;
......@@ -244,6 +246,62 @@ public class ManifestReaderTest {
Assert.assertEquals(counter.getCount(), 1);
}
@Test public void testKeyValueManifestFileWithPunctuations() {
TestManifestReader manifestReader = new TestManifestReader(new ManifestFieldDefinition.Builder()
.meta().required().name("FIELD_NAME_1").desc("some desc").and()
.meta().required().name("FIELD_NAME_2").desc("some desc").and()
.meta().required().name("FIELD_NAME_3").desc("some desc").build());
manifestReader.readManifest(Paths.get("."), new ManifestBuilder()
.field("fieldName1", "val1")
.field("field_name_2", "val2")
.field("field-name-3", "val3")
.build());
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_1").getValue(), "val1");
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_2").getValue(), "val2");
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_3").getValue(), "val3");
}
@Test public void testJsonManifestFileWithPunctuations() {
TestManifestReader manifestReader = new TestManifestReader(new ManifestFieldDefinition.Builder()
.meta().required().name("FIELD_NAME_1").desc("some desc").attributes(new ManifestFieldDefinition.Builder()
.attribute().optional().name("ATT_NAME_1").desc("some desc").build())
.and()
.meta().required().name("FIELD_NAME_2").desc("some desc").attributes(new ManifestFieldDefinition.Builder()
.attribute().optional().name("ATT_NAME_2").desc("some desc").build())
.and()
.meta().required().name("FIELD_NAME_3").desc("some desc").attributes(new ManifestFieldDefinition.Builder()
.attribute().optional().name("ATT_NAME_3").desc("some desc").build())
.build());
manifestReader.readManifest(Paths.get("."), new ManifestBuilder().jsonFormat()
.field("fieldName1", "val1").attribute("fieldName1", "attName1", "attval1")
.field("field_name_2", "val2").attribute("field_name_2", "att_name_2", "attval2")
.field("field-name-3", "val3").attribute("field-name-3", "att-name-3", "attval3")
.build());
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_1").getValue(), "val1");
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_1").getAttributes().get(0).getValue(), "attval1");
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_2").getValue(), "val2");
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_2").getAttributes().get(0).getValue(), "attval2");
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_3").getValue(), "val3");
Assert.assertEquals(
manifestReader.getManifestReaderResult().getField("FIELD_NAME_3").getAttributes().get(0).getValue(), "attval3");
}
private static File createManifest(String contents) {
try {
return Files.write(Files.createTempFile("TEMP", "MANIFEST"),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment