Unverified Commit b9b16bc4 authored by Mihaly Varadi's avatar Mihaly Varadi Committed by GitHub
Browse files

Merge pull request #3 from 3D-Beacons/ped

Adding PED and removing unused services
parents 66e9f60c 21dfeaaa
Pipeline #108818 passed with stage
in 7 seconds
[run]
omit =
resources/*
beacons_bio_3d/__main__.py
[report]
show_missing = True
\ No newline at end of file
name: Registry CI
on: [ push ]
# branches: [ master, development ]
# pull_request:
# branches: [ master, development ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
max-parallel: 4
matrix:
python-version: [3.8]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
make dev_deps
- name: Lint with flake8
run: |
make analyze
- name: Run Tests
run: |
make test
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# pyenv
.python-version
# dotenv
.env
# virtualenv
.venv
venv/
ENV/
# mkdocs documentation
/site
# mypy
.mypy_cache/
.vscode/
.idea/*
\ No newline at end of file
# Author: Sreenath Nair
# Email: sreenath@ebi.ac.uk
# Runner should have docker installed
stages:
- build
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
docker-build:
stage: build
image: docker/compose
script:
- docker build --tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} .
- docker push ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}
tags:
- pdbekb-shell
Changelog
---------
0.0.1
~~~~~
* Initial release.
# pull official base image
FROM python:3.8-slim-buster
ADD . /app
WORKDIR /app
RUN pip install -e .
include LICENSE
include README.md
include CHANGES.md
\ No newline at end of file
default: test
clean-pyc:
@find . -iname '*.py[co]' -delete
@find . -iname '__pycache__' -delete
clean-pytest:
rm -rf .pytest_cache
clean-dist:
@rm -rf dist/
@rm -rf build/
@rm -rf *.egg-info
clean: clean-pyc clean-pytest clean-dist
analyze:
# stop the build if there are Python syntax errors or undefined names
flake8 --count --select=E9,F63,F7,F82 --show-source --statistics ./beacons_bio_3d
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=migrations ./beacons_bio_3d
test:
pytest --cov-config=.coveragerc --cov-report=html --cov=beacons_bio_3d .
dist: clean
python3 setup.py sdist
python3 setup.py bdist_wheel
version: dist
python3 setup.py --version
license: dist
python3 setup.py --license
dev_deps:
pip install six jsonschema pytest-cov pytest-xdist flake8
1.0.0
\ No newline at end of file
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import json
import argparse
from beacons_bio_3d.utils import JSONUtils, DjangoUtils
__doc__ = """
3D Beacons
Registry schema validation and utilities.
"""
RES_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'resources')
DEFAULT_SCHEMA_JSON = f"{RES_PATH}/schema.json"
DEFAULT_REGISTRY_JSON = f"{RES_PATH}/registry.json"
def main():
""" This is the application entry point
"""
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
sub_parsers = parser.add_subparsers(dest='subparser_name', help="Available sub-commands")
# parser for validation functions
validate_parser = sub_parsers.add_parser("validate_schema", help="Validates the registry JSON with the defined schema.")
validate_parser.add_argument("--schema_json", help="Path to the schema JSON", required=True)
validate_parser.add_argument("--registry_json", help="Path to the registry JSON", required=True)
# parser for model generation functions
model_gen_parser = sub_parsers.add_parser("model_generate", help="Generates model JSON from registry JSON.")
model_gen_parser.add_argument("--registry_json", help="Path to registry JSON to be converted", required=True)
model_gen_parser.add_argument("--model_json", help="Path to output model JSON", required=True)
model_gen_parser.add_argument(
"--django_app",
help="The name of Django app, this will be used to define package for model",
default="core")
args = parser.parse_args()
if args.subparser_name == "validate_schema":
JSONUtils.validate_schema(args.schema_json, args.registry_json)
elif args.subparser_name == "model_generate":
if JSONUtils.validate_schema(DEFAULT_SCHEMA_JSON, args.registry_json):
# generate model json
gen_model = DjangoUtils.generate_fixture_json(args.registry_json, django_app=args.django_app)
with open(args.model_json, "w") as model_json_handle:
json.dump(gen_model, model_json_handle)
print(f"Model JSON generated to {args.model_json}")
if __name__ == "__main__":
main()
import os
import json
import jsonschema
from jsonschema.exceptions import ValidationError
class DjangoUtils:
""" This is the utils class for Django
"""
def generate_fixture_json(input_registry_json, django_app="core"):
registry = json.load(open(input_registry_json))
model = []
for provider in registry["providers"]:
model.append(
{
"model": f"{django_app}.Provider",
"pk": provider["providerId"],
"fields": {
"name": provider["providerName"],
"description": provider["providerDescription"],
"url": provider["providerUrl"],
"base_service_url": provider["baseServiceUrl"]
}
}
)
for service in registry["services"]:
model.append({
"model": f"{django_app}.Service",
"fields": {
"service_type": service["serviceType"],
"provider": service["provider"],
"access_point": service["accessPoint"]
}
})
return model
class JSONUtils:
""" This is the utils class for JSON
"""
def validate_schema(schema_json, registry_json):
""" Function to validate registry JSON against defined schema
Args:
schema_json (str): Path to schema JSON
registry_json (str): Path to registry JSON
Raises:
FileNotFoundError: Raised when registry or schema JSON files are not present
"""
if not os.path.exists(schema_json):
raise FileNotFoundError("Schema JSON does not exists")
if not os.path.exists(registry_json):
raise FileNotFoundError("Registry JSON does not exists")
schema = json.load(open(schema_json))
instance = json.load(open(registry_json))
try:
jsonschema.validate(instance, schema=schema)
except ValidationError as e:
raise e
# validate if provider field in services is valid
dict_provider = dict()
for provider in instance["providers"]:
dict_provider[provider["providerId"]] = True
unmatched_providers = []
for service in instance["services"]:
t_provider = service["provider"]
if not dict_provider.get(t_provider):
unmatched_providers.append(t_provider)
if unmatched_providers:
raise InvalidProviderException(unmatched_providers)
print("Validation successful")
return True
class InvalidProviderException(Exception):
""" Exception raised when provider id in services sections are not present in providers list
Attributes:
providers -- List of invalid providers found
"""
def __init__(self, providers):
self.providers = providers
super().__init__(f"Invalid providers: {self.providers}")
{
"providers": [
{
"providerId": "swiss-model",
"providerName": "SWISS-MODEL",
"providerDescription": "SWISS-MODEL is a fully automated protein structure homology-modelling server, accessible via the ExPASy web server, or from the program DeepView (Swiss Pdb-Viewer). The purpose of this server is to make protein modelling accessible to all life science researchers worldwide.",
"providerUrl": "https://swissmodel.expasy.org/",
"baseServiceUrl": "https://swissmodel.expasy.org/beacons/"
},
{
"providerId": "genome-3d",
"providerName": "Genome-3D",
"providerDescription": "Genome3D provides consensus structural annotations and 3D models for sequences from model organisms, including human.",
"providerUrl": "http://genome3d.eu/",
"baseServiceUrl": "http://genome3d.eu/beacons/"
},
{
"providerId": "pdbe",
"providerName": "Protein Databank in Europe",
"providerDescription": "PDBe is the European resource for the collection, organisation and dissemination of data on biological macromolecular structures.",
"providerUrl": "https://www.ebi.ac.uk/pdbe/",
"baseServiceUrl": "https://www.ebi.ac.uk/pdbe/beacons/"
},
{
"providerId": "pdbe-kb",
"providerName": "Protein Data Bank in Europe - Knowledge Base",
"providerDescription": "PDBe-KB (Protein Data Bank in Europe - Knowledge Base) is a community-driven resource managed by the PDBe team, collating functional annotations and predictions for structure data in the PDB archive. PDBe-KB is a collaborative effort between PDBe and a diverse group of bioinformatics resources and research teams.",
"providerUrl": "https://www.ebi.ac.uk/pdbe/pdbe-kb",
"baseServiceUrl": "https://www.ebi.ac.uk/pdbe/pdbe-kb/beacons/"
}
],
"services": {
"uniprot": {
"providers": {
"swiss-model": {
"accessPoint": "uniprot/structures"
},
"genome-3d": {
"accessPoint": "uniprot/structures"
},
"pdbe": {
"accessPoint": "uniprot/structures"
},
"pdbe-kb": {
"accessPoint": "uniprot/structures"
}
}
},
"sequence": {
"providers": {
"swiss-model": {
"accessPoint": "sequence/structures"
},
"genome-3d": {
"accessPoint": "sequence/structures"
},
"pdbe": {
"accessPoint": "sequence/structures"
},
"pdbe-kb": {
"accessPoint": "sequence/structures"
}
}
},
"taxonomy": {
"providers": {
"swiss-model": {
"accessPoint": "taxonomy/structures"
},
"genome-3d": {
"accessPoint": "taxonomy/structures"
},
"pdbe": {
"accessPoint": "taxonomy/structures"
},
"pdbe-kb": {
"accessPoint": "taxonomy/structures"
}
}
}
}
}
\ No newline at end of file
{
"providers": [
{
"providerId": "swissmodel",
"providerName": "SWISS-MODEL",
"providerDescription": "SWISS-MODEL is a fully automated protein structure homology-modelling server, accessible via the ExPASy web server, or from the program DeepView (Swiss Pdb-Viewer). The purpose of this server is to make protein modelling accessible to all life science researchers worldwide.",
"providerUrl": "https://swissmodel.expasy.org/",
"baseServiceUrl": "https://beta.swissmodel.expasy.org/3d-beacons/",
"providerLogo": "https://swissmodel.expasy.org"
},
{
"providerId": "genome3d",
"providerName": "Genome-3D",
"providerDescription": "Genome3D provides consensus structural annotations and 3D models for sequences from model organisms, including human.",
"providerUrl": "http://genome3d.eu/",
"baseServiceUrl": "http://genome3d.eu/beacons/",
"providerLogo": "http://genome3d.eu/"
},
{
"providerId": "pdb",
"providerName": "Protein Data Bank in Europe",
"providerDescription": "PDBe is the European resource for the collection, organisation and dissemination of data on biological macromolecular structures.",
"providerUrl": "https://www.ebi.ac.uk/pdbe/",
"baseServiceUrl": "https://wwwdev.ebi.ac.uk/pdbe/aggregated-api/beacons/",
"providerLogo": "https://www.ebi.ac.uk/pdbe/static/images/logos/PDBe/logo.png"
},
{
"providerId": "ped",
"providerName": "Protein Ensemble Database",
"providerDescription": "The Protein Ensemble Database (PED) is an open access database for the deposition of structural ensembles, mainly intrinsically disordered proteins (IDPs).",
"providerUrl": "https://proteinensemble.org/",
"baseServiceUrl": "https://proteinensemble.org/api/3d-beacons/",
"providerLogo": "https://proteinensemble.org/assets/PED_logo.svg"
}
],
"services": [
{
"serviceType": "uniprot",
"provider": "swissmodel",
"accessPoint": "uniprot/"
},
{
"serviceType": "uniprot",
"provider": "genome3d",
"accessPoint": "uniprot/structures"
},
{
"serviceType": "uniprot",
"provider": "pdb",
"accessPoint": "uniprot/"
},
{
"serviceType": "uniprot",
"provider": "ped",
"accessPoint": ""
}
]
}
\ No newline at end of file
{
"$schema": "https://json-schema.org/draft/2019-09/schema",
"$id": "https://json-schema.org/draft/2019-09/schema",
"type": "object",
"title": "The registry schema",
"description": "The JSON schema for registry document.",
"required": [
"providers",
"services"
],
"properties": {
"providers": {
"$id": "#/properties/providers",
"type": "array",
"title": "Providers list",
"description": "A list of providers objects",
"items": [
{
"type": "object",
"required": [
"providerId",
"providerName",
"providerDescription",
"providerUrl",
"baseServiceUrl",
"providerLogo"
],
"properties": {
"providerId": {
"$id": "#/properties/providers/properties/providerId",
"type": "string",
"title": "Provider ID",
"description": "A unique id for provider",
"examples": [
"pdbe"
]
},
"providerName": {
"$id": "#/properties/providers/properties/providerName",
"type": "string",
"title": "Provider name",
"description": "A name for provider",
"examples": [
"Protein Databank in Europe"
]
},
"providerDescription": {
"$id": "#/properties/providers/properties/providerDescription",
"type": "string",
"title": "Provider description",
"description": "A description for provider",
"examples": [
"PDBe is the European resource for the collection, organisation and dissemination of data on biological macromolecular structures."
]
},
"providerUrl": {
"$id": "#/properties/providers/properties/providerUrl",
"type": "string",
"title": "Provider url",
"description": "A url to the provider website",
"examples": [
"https://www.ebi.ac.uk/pdbe/"
]
},
"baseServiceUrl": {
"$id": "#/properties/providers/properties/baseServiceUrl",
"type": "string",
"title": "Base service url",
"description": "A base url for the service of each provider",
"examples": [
"https://www.ebi.ac.uk/pdbe/beacons/"
],
"default": ""
},
"providerLogo": {
"$id": "#/properties/providers/properties/providerLogo",
"type": "string",
"title": "Provider logo URL",
"description": "A url to the provider logo",
"examples": [
"https://www.ebi.ac.uk/pdbe/docs_dev/logos/images/RGB/PDBe-letterhead-white-RGB_2013.png"
],
"default": ""
}
}
}
]
},
"services": {
"$id": "#/properties/services",
"type": "array",
"title": "Services list",
"description": "A list of services objects.",
"items": [
{
"type": "object",
"required": [
"serviceType",
"provider",
"accessPoint"
],
"properties": {
"serviceType": {
"$id": "#/properties/services/properties/serviceType",
"type": "string",
"title": "Service type",