Unverified Commit c4d86a7e authored by Isuru Liyanage's avatar Isuru Liyanage Committed by GitHub
Browse files

Merge pull request #38 from elixir-europe/dev

Release v1.10.0
parents 8ef80877 4e64146f
......@@ -2,4 +2,5 @@ node_modules/
.idea/
.DS_Store
log/
*.iml
\ No newline at end of file
*.iml
server.pid
{
"sample_labels": [
{
"label": "test1"
},
{
"label": "test2"
}
]
}
{
"$schema": "https://json-schema.org/draft/2019-09/schema",
"$id": "test_schema_X",
"type": "object",
"required": ["sample_labels"],
"additionalProperties": false,
"properties": {
"sample_labels": {
"type": "array",
"minItems": 1,
"additionalProperties": false,
"items": {
"type": "object",
"required": ["label"],
"additionalProperties": false,
"properties": {
"label": {
"type": "string"
}
}
}
}
}
}
This diff is collapsed.
{
"name": "biovalidator",
"version": "1.9.1",
"version": "1.10.0",
"description": "A nodejs JSON schema validator service. Extended from Ajv to include ontology validation.",
"main": "src/server.js",
"repository": "https://github.com/elixir-europe/biovalidator.git",
......@@ -19,12 +19,13 @@
"schema",
"validator"
],
"author": "EMBL-EBI-SUBS, fpenim, ke4, haseeb-gh",
"author": "EMBL-EBI-AIT, fpenim, ke4, haseeb-gh",
"license": "Apache-2.0",
"dependencies": {
"ajv": "~7.2.1",
"ajv-formats": "^1.5.1",
"express": "^4.17.1",
"glob": "^8.0.3",
"npid": "^0.4.0",
"request": "^2.88.2",
"request-promise": "^4.2.4",
......@@ -34,7 +35,6 @@
},
"devDependencies": {
"bufferutil": "^4.0.2",
"canvas": "^2.6.1",
"jest": "^26.6.0",
"nodemon": "^2.0.6",
"ts-node": "^9.1.0",
......
const runValidation = require("../validator");
const logger = require("../winston");
const logger = require("./winston");
const fs = require("fs");
const {log_error, log_info } = require("../utils/logger");
const {log_error, log_info } = require("./utils/logger");
const BioValidator = require("./biovalidator");
class BioValidatorCLI {
constructor(pathToSchema, pathToJson) {
constructor(pathToSchema, pathToJson, pathToRefSchema) {
this.pathToSchema = pathToSchema
this.pathToJson = pathToJson
this.biovalidator = new BioValidator(pathToRefSchema);
}
read_schema(pathToSchema) {
......@@ -32,8 +33,9 @@ class BioValidatorCLI {
validate() {
this.inputSchema = this.read_schema(this.pathToSchema)
this.jsonToValidate = this.read_json(this.pathToJson)
if (this.inputSchema && this.jsonToValidate) {
runValidation(this.inputSchema, this.jsonToValidate).then((output) => {
this.biovalidator.runValidation(this.inputSchema, this.jsonToValidate).then((output) => {
logger.log("silly", "Sent validation results.");
this.process_output(output);
}).catch((error) => {
......
......@@ -5,28 +5,60 @@
const Promise = require('bluebird');
const path = require("path");
const fs = require('fs');
const Ajv = require("ajv").default;
const Ajv = require("ajv/dist/2019").default;
const addFormats = require("ajv-formats");
const request = require("request-promise");
const AppError = require("./model/application-error");
const {getFiles, readFile} = require("./utils/file_utils");
const {isChildTermOf, isValidTerm, isValidTaxonomy} = require("./keywords");
const GraphRestriction = require("./keywords/graph_restriction");
const ValidationError = require("./model/validation-error");
const logger = require("./winston");
const devMode = 0;
console.debug = devMode ? console.debug : () => { };
console.debug = devMode ? console.debug : () => {
};
/**
*
* Wraps the generic validator, outputs errors in custom format.
*
*/
const customKeywordValidators = [
new isChildTermOf(null, "https://www.ebi.ac.uk/ols/api/search?q="),
new isValidTerm(null, "https://www.ebi.ac.uk/ols/api/search?q="),
new isValidTaxonomy(null),
new GraphRestriction(null, "https://www.ebi.ac.uk/ols/api")
];
class BioValidator {
constructor(customKeywordValidators, baseSchemaPath){
constructor(localSchemaPath) {
this.validatorCache = {};
this.cachedSchemas = {};
this.ajvInstance = this.constructAjv(customKeywordValidators);
this.baseSchemaPath = baseSchemaPath;
this.customKeywordValidators = customKeywordValidators;
this.localSchemaPath = localSchemaPath;
this.ajvInstance = this.constructAjv();
}
runValidation(inputSchema, inputObject) {
logger.log("silly", "Running validation...");
return new Promise((resolve, reject) => {
this.validate(inputSchema, inputObject)
.then((validationResult) => {
if (validationResult.length === 0) {
resolve([]);
} else {
let ajvErrors = [];
validationResult.forEach(validationError => {
ajvErrors.push(validationError);
});
resolve(this.convertToValidationErrors(ajvErrors));
}
}
).catch((error) => {
if (error.errors) {
reject(new AppError(error.errors));
} else {
logger.log("error", "An error ocurred while running the validation. Error : " + JSON.stringify(error));
reject(new AppError("An error ocurred while running the validation."));
}
});
});
}
validate(inputSchema, inputObject) {
......@@ -60,13 +92,23 @@ class BioValidator {
});
}
validateWithRemoteSchema(schemaUri, document) {
return this.getSchema(schemaUri)
.then(schema => {return this.validateSingleSchema(document, schema)})
convertToValidationErrors(ajvErrorObjects) {
let localErrors = [];
ajvErrorObjects.forEach( (errorObject) => {
let tempValError = new ValidationError(errorObject);
let index = localErrors.findIndex(valError => (valError.dataPath === tempValError.dataPath));
if(index !== -1) {
localErrors[index].errors.push(tempValError.errors[0]);
} else {
localErrors.push(tempValError);
}
});
return localErrors;
}
getSchema(schemaUri) {
if(! this.cachedSchemas[schemaUri]) {
if (!this.cachedSchemas[schemaUri]) {
return new Promise((resolve, reject) => {
BioValidator.fetchSchema(schemaUri)
.then(schema => {
......@@ -93,26 +135,30 @@ class BioValidator {
getValidationFunction(inputSchema) {
const schemaId = inputSchema['$id'];
if(this.validatorCache[schemaId]) {
if (this.validatorCache[schemaId]) {
return Promise.resolve(this.validatorCache[schemaId]);
} else {
const compiledSchemaPromise = this.ajvInstance.compileAsync(inputSchema);
if(schemaId) {
if (schemaId) {
this.validatorCache[schemaId] = compiledSchemaPromise;
}
return Promise.resolve(compiledSchemaPromise);
}
}
constructAjv(customKeywordValidators) {
const ajvInstance = new Ajv({allErrors: true, strict:false, loadSchema: this.generateLoadSchemaRefFn()});
constructAjv() {
const ajvInstance = new Ajv({allErrors: true, strict: false, loadSchema: this.generateLoadSchemaRefFn()});
const draft7MetaSchema = require("ajv/dist/refs/json-schema-draft-07.json")
ajvInstance.addMetaSchema(draft7MetaSchema)
addFormats(ajvInstance);
BioValidator._addCustomKeywordValidators(ajvInstance, customKeywordValidators);
this.addCustomKeywordValidators(ajvInstance);
this.preCompileLocalSchemas(ajvInstance);
return ajvInstance
}
static _addCustomKeywordValidators(ajvInstance, customKeywordValidators) {
addCustomKeywordValidators(ajvInstance) {
customKeywordValidators.forEach(customKeywordValidator => {
ajvInstance = customKeywordValidator.configure(ajvInstance);
});
......@@ -122,41 +168,39 @@ class BioValidator {
generateLoadSchemaRefFn() {
const cachedSchemas = this.cachedSchemas;
const baseSchemaPath = this.baseSchemaPath;
const loadSchemaRefFn = (uri) => {
if(cachedSchemas[uri]) {
return (uri) => {
if (cachedSchemas[uri]) {
return Promise.resolve(cachedSchemas[uri]);
} else {
if (baseSchemaPath) {
let ref = path.join(baseSchemaPath, uri);
console.log('loading ref ' + ref);
let jsonSchema = fs.readFileSync(ref);
let loadedSchema = JSON.parse(jsonSchema);
loadedSchema["$async"] = true;
cachedSchemas[uri] = loadedSchema;
return Promise.resolve(loadedSchema);
}
else {
return new Promise((resolve, reject) => {
request({
method: "GET",
url: uri,
json: true
}).then(resp => {
const loadedSchema = resp;
loadedSchema["$async"] = true;
cachedSchemas[uri] = loadedSchema;
resolve(loadedSchema);
}).catch(err => {
reject(err);
});
return new Promise((resolve, reject) => {
request({
method: "GET",
url: uri,
json: true
}).then(resp => {
const loadedSchema = resp;
loadedSchema["$async"] = true;
cachedSchemas[uri] = loadedSchema;
resolve(loadedSchema);
}).catch(err => {
console.log("Failed to retrieve remote schema: " + uri)
reject(err);
});
}
});
}
};
}
return loadSchemaRefFn;
preCompileLocalSchemas(ajv) {
if (this.localSchemaPath) {
let schemaFiles = getFiles(this.localSchemaPath);
for (let file of schemaFiles) {
let schema = readFile(file);
ajv.getSchema(schema["$id"] || ajv.compile(schema));
this.cachedSchemas[schema["$id"]] = schema;
}
}
}
}
......
const express = require("express");
const bodyParser = require("body-parser");
const logger = require("./winston");
const runValidation = require("./validator");
const AppError = require("./model/application-error");
const BioValidator = require("./biovalidator")
const argv = require("yargs").argv;
const npid = require("npid");
const app = express();
const router = express.Router()
const port = process.env.PORT || 3020;
const basePath = process.env.VALIDATOR_BASE_URL || '/';
app.use(express.static('src/views'));
const biovalidator = new BioValidator();
router.use(express.static('src/views'));
app.use(bodyParser.json());
......@@ -32,14 +36,14 @@ app.use(function(err, req, res, next) {
}
});
app.post("/validate", (req, res) => {
router.post("/validate", (req, res) => {
logger.log("debug", "Received POST request.");
var inputSchema = req.body.schema;
var inputObject = req.body.object;
let inputSchema = req.body.schema;
let inputObject = req.body.object;
if (inputSchema && inputObject) {
runValidation(inputSchema, inputObject).then((output) => {
biovalidator.runValidation(inputSchema, inputObject).then((output) => {
logger.log("silly", "Sent validation results.");
res.status(200).send(output);
}).catch((error) => {
......@@ -54,7 +58,7 @@ app.post("/validate", (req, res) => {
}
});
app.get("/validate", (req, res) => {
router.get("/validate", (req, res) => {
logger.log("silly", "Received GET request.");
res.send({
......@@ -66,6 +70,8 @@ app.get("/validate", (req, res) => {
});
});
app.use(basePath, router);
app.listen(port, () => {
logger.log("info", ` -- Started server on port ${port} --`);
if(argv.logPath) { logger.log("info", ` --> Log output: ${argv.logPath}`); }
......
const fs = require('fs');
const glob = require('glob');
function getFiles(filePattern) {
let files = [];
if (Array.isArray(filePattern)) {
for (let fp of filePattern) {
addFiles(fp, files);
}
} else {
addFiles(filePattern, files);
}
return new Set(files);
}
function readFile(filePath) {
let schema = fs.readFileSync(filePath).toString();
let jsonSchema = JSON.parse(schema.toString());
jsonSchema["$async"] = true;
return jsonSchema;
}
function addFiles(filePattern, files) {
if (glob.hasMagic(filePattern)) {
const dataFiles = glob.sync(filePattern, {cwd: process.cwd()})
files.push(...dataFiles);
} else {
if (fs.lstatSync(filePattern).isDirectory()) {
fs.readdirSync(filePattern).forEach(file => {
files.push(filePattern + "/" + file);
});
} else {
files.push(filePattern);
}
}
}
module.exports = {getFiles, readFile}
const logger = require("./winston");
const ValidationError = require("./model/validation-error");
const AppError = require("./model/application-error");
const BioValidator = require("./bio-validator");
const GraphRestriction = require("./keywords/graph_restriction");
const { isChildTermOf, isValidTerm, isValidTaxonomy } = require("./keywords");
const validator = new BioValidator([
new isChildTermOf(null, "https://www.ebi.ac.uk/ols/api/search?q="),
new isValidTerm(null, "https://www.ebi.ac.uk/ols/api/search?q="),
new isValidTaxonomy(null),
new GraphRestriction(null, "https://www.ebi.ac.uk/ols/api")
]);
function convertToValidationErrors(ajvErrorObjects) {
let localErrors = [];
ajvErrorObjects.forEach( (errorObject) => {
let tempValError = new ValidationError(errorObject);
let index = localErrors.findIndex(valError => (valError.dataPath === tempValError.dataPath));
if(index !== -1) {
localErrors[index].errors.push(tempValError.errors[0]);
} else {
localErrors.push(tempValError);
}
});
return localErrors;
}
function runValidation(inputSchema, inputObject) {
logger.log("silly", "Running validation...");
return new Promise((resolve, reject) => {
validator.validate(inputSchema, inputObject)
.then((validationResult) => {
if (validationResult.length === 0) {
resolve([]);
} else {
let ajvErrors = [];
validationResult.forEach(validationError => {
ajvErrors.push(validationError);
});
resolve(convertToValidationErrors(ajvErrors));
}
}
).catch((error) => {
if (error.errors) {
reject(new AppError(error.errors));
} else {
logger.log("error", "An error ocurred while running the validation. Error : " + JSON.stringify(error));
reject(new AppError("An error ocurred while running the validation."));
}
});
});
}
module.exports = runValidation;
const fs = require("fs");
const runValidation = require("../src/validator");
const BioValidator = require("../src/biovalidator")
const biovalidator = new BioValidator();
test(" -> isChildTermOf Schema", () => {
let inputSchema = fs.readFileSync("examples/schemas/isChildTerm-schema.json", "utf-8");
......@@ -8,7 +9,7 @@ test(" -> isChildTermOf Schema", () => {
let inputObj = fs.readFileSync("examples/objects/isChildTerm.json", "utf-8");
let jsonObj = JSON.parse(inputObj);
return runValidation(jsonSchema, jsonObj).then( (data) => {
return biovalidator.runValidation(jsonSchema, jsonObj).then( (data) => {
expect(data).toBeDefined();
expect(data[0]).toBeDefined();
expect(data[0].dataPath).toBe("/attributes/age/0/terms/0/url");
......@@ -22,7 +23,7 @@ test("FAANG Schema - FAANG \'organism\' sample", () => {
let inputObj = fs.readFileSync("examples/objects/faang-organism-sample.json", "utf-8");
let jsonObj = JSON.parse(inputObj);
return runValidation(jsonSchema, jsonObj).then( (data) => {
return biovalidator.runValidation(jsonSchema, jsonObj).then( (data) => {
expect(data).toBeDefined();
expect(data.length).toBe(0);
});
......@@ -35,7 +36,7 @@ test("FAANG Schema - \'specimen\' sample", () => {
let inputObj = fs.readFileSync("examples/objects/faang-specimen-sample.json", "utf-8");
let jsonObj = JSON.parse(inputObj);
return runValidation(jsonSchema, jsonObj).then( (data) => {
return biovalidator.runValidation(jsonSchema, jsonObj).then( (data) => {
expect(data).toBeDefined();
expect(data.length).toBe(0);
});
......@@ -48,7 +49,7 @@ test("FAANG Schema - \'pool of specimens\' sample", () => {
let inputObj = fs.readFileSync("examples/objects/faang-poolOfSpecimens-sample.json", "utf-8");
let jsonObj = JSON.parse(inputObj);
return runValidation(jsonSchema, jsonObj).then( (data) => {
return biovalidator.runValidation(jsonSchema, jsonObj).then( (data) => {
expect(data).toBeDefined();
expect(data.length).toBe(0);
});
......@@ -61,7 +62,7 @@ test("FAANG Schema - \'cell specimen\' sample", () => {
let inputObj = fs.readFileSync("examples/objects/faang-cellSpecimen-sample.json", "utf-8");
let jsonObj = JSON.parse(inputObj);
return runValidation(jsonSchema, jsonObj).then( (data) => {
return biovalidator.runValidation(jsonSchema, jsonObj).then( (data) => {
expect(data).toBeDefined();
expect(data.length).toBe(0);
});
......@@ -74,7 +75,7 @@ test("FAANG Schema - \'cell culture\' sample", () => {
let inputObj = fs.readFileSync("examples/objects/faang-cellCulture-sample.json", "utf-8");
let jsonObj = JSON.parse(inputObj);
return runValidation(jsonSchema, jsonObj).then( (data) => {
return biovalidator.runValidation(jsonSchema, jsonObj).then( (data) => {
expect(data).toBeDefined();
expect(data.length).toBe(0);
});
......@@ -87,7 +88,7 @@ test("FAANG Schema - \'cell line\' sample", () => {
let inputObj = fs.readFileSync("examples/objects/faang-cellLine-sample.json", "utf-8");
let jsonObj = JSON.parse(inputObj);
return runValidation(jsonSchema, jsonObj).then( (data) => {
return biovalidator.runValidation(jsonSchema, jsonObj).then( (data) => {
expect(data).toBeDefined();
expect(data.length).toBe(0);
});
......
const BioValidatorCLI = require("../src/cli/bio-validator-cli")
const BioValidatorCLI = require("../src/biovalidator-cli")
test("Using wrong parameters results with error", () => {
const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
......@@ -43,3 +43,15 @@ test( "Invalid JSON should result with validation error", () => {
expect(errorOutput).toEqual(expectedErrorOutput)
});
test("Should be able to reference schemas from a directory", () => {
const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
const schema = "test/resources/ref_test_schema.json";
const data = "test/resources/ref_test_valid.json";
const ref = "test/resources/schema_dir/*";
const cli = new BioValidatorCLI(schema, data, ref);
cli.validate();
});
const fs = require("fs");
const BioValidator = require('../src/bio-validator');
const BioValidator = require('../src/biovalidator');
test("Empty Schema (empty object)", () => {
const validator = new BioValidator([]);
const validator = new BioValidator();
return validator.validate({}, {}).then( (data) => {
expect(data).toBeDefined();
expect(data.length).toBe(0);
......@@ -16,7 +16,7 @@ test("Attributes Schema", () => {
let inputObj = fs.readFileSync("examples/objects/attributes.json");
let jsonObj = JSON.parse(inputObj);
const validator = new BioValidator([]);
const validator = new BioValidator();
return validator.validate(jsonSchema, jsonObj).then((data) => {
expect(data).toBeDefined();
......@@ -33,7 +33,7 @@ test("BioSamples Schema - FAANG \'organism\' sample", () => {
let inputObj = fs.readFileSync("examples/objects/faang-organism-sample.json");
let jsonObj = JSON.parse(inputObj);
const validator = new BioValidator([]);
const validator = new BioValidator();
return validator.validate(jsonSchema, jsonObj).then((data) => {
expect(data).toBeDefined();
......@@ -47,7 +47,7 @@ test("Study Schema", () => {
let inputObj = fs.readFileSync("examples/objects/study.json");
let jsonObj = JSON.parse(inputObj);
const validator = new BioValidator([]);
const validator = new BioValidator();
return validator.validate(jsonSchema, jsonObj).then((data) => {
expect(data).