Commit 8e43961b authored by David Mendez's avatar David Mendez
Browse files

Fix pylint errors

parent f21fc8c3
......@@ -13,12 +13,16 @@ from app.namespaces.job_submission.submit_connectivity_controller import API as
from app.namespaces.job_submission.submit_blast_controller import API as submit_blast_search_namespace
from app.namespaces.job_statistics.record_search_controller import API as record_search_namespace
from app.namespaces.job_statistics.record_download_controller import API as record_download_namespace
from app.db import db
from app.db import DB
from app.config import RUN_CONFIG
from app.config import RunEnvs
def create_app():
"""
Creates the flask app
:return: Delayed jobs flask app
"""
flask_app = Flask(__name__)
flask_app.config['SQLALCHEMY_DATABASE_URI'] = RUN_CONFIG.get('sql_alchemy').get('database_uri')
......@@ -39,11 +43,11 @@ def create_app():
}
with flask_app.app_context():
db.init_app(flask_app)
DB.init_app(flask_app)
create_tables = RUN_CONFIG.get('sql_alchemy').get('create_tables', False)
if create_tables:
db.create_all()
DB.create_all()
api = Api(
title='ChEMBL Interface Delayed Jobs',
......@@ -54,8 +58,10 @@ def create_app():
authorizations=authorizations
)
for namespace in [job_admin_namespace, job_status_namespace, submit_test_job_namespace, submit_similarity_search_namespace, submit_substructure_search_namespace,
submit_connectivity_search_namespace, submit_blast_search_namespace, record_search_namespace, record_download_namespace]:
for namespace in [job_admin_namespace, job_status_namespace, submit_test_job_namespace,
submit_similarity_search_namespace, submit_substructure_search_namespace,
submit_connectivity_search_namespace, submit_blast_search_namespace, record_search_namespace,
record_download_namespace]:
api.add_namespace(namespace)
return flask_app
......
"""
Module that handles decorators used in the authorisation of different endpoints.
"""
from functools import wraps
from flask import request, jsonify
from app.config import RUN_CONFIG
import jwt
from app.config import RUN_CONFIG
def token_required_for_job_id(f):
@wraps(f)
# pylint: disable=W0702
def token_required_for_job_id(func):
"""
Checks the token provided, the job_id in the token must match the job id that the function aims receives as
parameter. Makes the function return a 403 http error if the token is missing, 401 if is invalid.
:param func: function to decorate
:return: decorated function
"""
@wraps(func)
def decorated(*args, **kwargs):
id = kwargs.get('id')
job_id = kwargs.get('id')
token = request.headers.get('X-Job-Key')
key = RUN_CONFIG.get('server_secret_key')
......@@ -17,23 +29,29 @@ def token_required_for_job_id(f):
try:
token_data = jwt.decode(token, key, algorithms=['HS256'])
authorised_id = token_data.get('job_id')
if authorised_id != job_id:
return jsonify({'message': f'You are not authorised modify the job {id}'}), 401
except:
return jsonify({'message': 'Token is invalid'}), 401
authorised_id = token_data.get('job_id')
if authorised_id != id:
return jsonify({'message': f'You are not authorised modify the job {id}'}), 401
return f(*args, **kwargs)
return func(*args, **kwargs)
return decorated
def admin_token_required(f):
@wraps(f)
def admin_token_required(func):
"""
Checks that a valid admin token is provided.
parameter. Makes the function return a 403 http error if the token is missing, 401 if is invalid.
:param func: function to decorate
:return: decorated function
"""
@wraps(func)
def decorated(*args, **kwargs):
username = kwargs.get('username')
token = request.headers.get('X-Admin-Key')
key = RUN_CONFIG.get('server_secret_key')
......@@ -42,12 +60,13 @@ def admin_token_required(f):
try:
token_data = jwt.decode(token, key, algorithms=['HS256'])
username = token_data.get('username')
if username != RUN_CONFIG.get('admin_username'):
return jsonify({'message': f'You are not authorised for this operation'}), 401
except:
return jsonify({'message': 'Token is invalid'}), 401
if username == RUN_CONFIG.get('admin_username'):
return jsonify({'message': f'You are not authorised for this operation'}), 401
return func(*args, **kwargs)
return f(*args, **kwargs)
return decorated
\ No newline at end of file
return decorated
"""
Module that handles the generation of tokens for the app
"""
import datetime
from app.config import RUN_CONFIG
import jwt
from app.config import RUN_CONFIG
JOB_TOKEN_HOURS_TO_LIVE = 24
ADMIN_TOKEN_HOURS_TO_LIVE=1
ADMIN_TOKEN_HOURS_TO_LIVE = 1
def generate_job_token(job_id):
"""
Generates a token that is valid ONLY to modify the job whose id is set as parameter.
:param job_id: id of the job for which the toke will be valid
:return: JWT token
"""
token_data = {
'job_id': job_id,
......@@ -20,6 +31,10 @@ def generate_job_token(job_id):
def generate_admin_token():
"""
Generates a token that can be used to be authorised for admin tasks
:return: JWT token
"""
token_data = {
'username': RUN_CONFIG.get('admin_username'),
......
"""
Module that handles the configuration of the app
"""
import os
import yaml
from pathlib import Path
import hashlib
from enum import Enum
import yaml
CUSTOM_CONFIG_FILE_PATH = os.getenv('CONFIG_FILE_PATH')
if CUSTOM_CONFIG_FILE_PATH is not None:
......@@ -14,20 +19,40 @@ print('CONFIG_FILE_PATH: ', CONFIG_FILE_PATH)
print('------------------------------------------------------------------------------------------------')
class RunEnvs(object):
class RunEnvs(Enum):
"""
Class that defines the possible run environments
"""
DEV = 'DEV'
TEST = 'TEST'
STAGING = 'STAGING'
PROD = 'PROD'
def __repr__(self):
return self.name
def __str__(self):
return self.name
def hash_secret(secret):
"""
Returns a a digest of a secret you want to store in memory
:param secret: secret you want to hash
:return: a sha256 hash of the secret, encoded in hexadecimal
"""
hashed = hashlib.sha256(secret.encode('UTF-8')).hexdigest()
return hashed
def verify_secret(prop_name, value):
"""
Verifies that a value in the current config (hashed) corresponds to the value passed as parameter (unhashed)
:param prop_name: name of the property in the configuration
:param value: clear text value of the property.
:return: True if the value is correct, false otherwise
"""
hashed = hashlib.sha256(value.encode('UTF-8')).hexdigest()
has_must_be = RUN_CONFIG.get(prop_name)
......@@ -38,4 +63,3 @@ RUN_CONFIG = yaml.load(open(CONFIG_FILE_PATH, 'r'), Loader=yaml.FullLoader)
# Hash keys and passwords
RUN_CONFIG['admin_password'] = hash_secret(RUN_CONFIG.get('admin_password'))
"""
Module that handles the connection with the database
"""
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
DB = SQLAlchemy()
"""
Module that handles the connection with elasticsearch
"""
from elasticsearch import Elasticsearch
from app.config import RUN_CONFIG
es = Elasticsearch([RUN_CONFIG.get('elasticsearch').get('host')])
ES = Elasticsearch([RUN_CONFIG.get('elasticsearch').get('host')])
#!/usr/bin/env python3
"""
Script that runs the functional tests for the app
"""
import argparse
# pylint: disable=E0401
import test_successful_job_run
import test_job_cache
import test_parallel_job_submission
import test_failing_job
import test_output_file_lost
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('server_base_path', help='server base path to run the tests against',
PARSER = argparse.ArgumentParser()
PARSER.add_argument('server_base_path', help='server base path to run the tests against',
default='http://127.0.0.1:5000', nargs='?')
args = parser.parse_args()
ARGS = PARSER.parse_args()
def run():
"""
Runs all functional tests
"""
for test_module in [test_successful_job_run, test_job_cache, test_parallel_job_submission, test_failing_job,
test_output_file_lost]:
test_module.run_test(args.server_base_path)
test_module.run_test(ARGS.server_base_path)
if __name__ == "__main__":
run()
import requests
"""
Tests that a failing job is restarted up to n times when it is submitted again
"""
import time
import datetime
import requests
# pylint: disable=R0914
# pylint: disable=R0915
def run_test(server_base_url):
"""
Submits a job that will always fail and tests that it is restarted when submitted again. But only up to n times
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
"""
print('------------------------------------------------------------------------------------------------')
print('Going to test the a job that always fails')
......@@ -80,5 +90,3 @@ def run_test(server_base_url):
print(f'started_at_1: {started_at_1}')
assert started_at_0 == started_at_1, 'The job must have not started again, the max retries limit was reached.'
import requests
"""
Module that runs tests for job caching
"""
import time
import datetime
import requests
# pylint: disable=R0914
def run_test(server_base_url):
"""
Test that when a job that is submitted is has exactly the same parameters of a previously run job, the job is not
run again. It just returns the existing job.
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
"""
print('------------------------------------------------------------------------------------------------')
print('Going to test the job caching')
......
import requests
"""
Module that runs tests when an output file of a job goes missing
"""
import time
import datetime
import requests
def run_test(server_base_url):
"""
Tests that when an output file of a job is missing, the job is started again.
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
"""
print('------------------------------------------------------------------------------------------------')
print('Going to test the lost of a results file')
......
import requests
"""
Module that runs tests for parallel job submission
"""
import time
import datetime
import requests
def run_test(server_base_url):
"""
Submits a job, and while it is running it submits another job with exactly the same parameters. No new job should be
started, it should return the id for the job that is already running without restarting it.
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
"""
print('------------------------------------------------------------------------------------------------')
print('Going to test the when a job is submitted while the same job is already running')
......
import requests
"""
Module that runs a normal simple job and expects it to run correctly.
"""
import time
import os
import requests
def run_test(server_base_url):
"""
Tests that a job can run normally.
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
"""
print('------------------------------------------------------------------------------------------------')
print('Going to test a successful job run')
......@@ -54,8 +61,3 @@ def run_test(server_base_url):
print(f'full_output_file_url: {full_output_file_url}')
file_request = requests.get(full_output_file_url)
assert file_request.status_code == 200, 'The results file could not be downloaded!!'
from flask import abort, request, make_response, jsonify
"""
Module that describes and handles the requests concerned with performing admin tasks
"""
from flask import request, make_response, jsonify
from flask_restplus import Namespace, Resource, fields
from app.authorisation import token_generator
from app.config import verify_secret
from app.config import RUN_CONFIG
......@@ -13,6 +17,7 @@ OPERATION_RESULT = API.model('OperationResult', {
})
# pylint: disable=no-self-use,broad-except
@API.route('/login')
class AdminLogin(Resource):
"""
......@@ -56,8 +61,8 @@ class DeleteExpired(Resource):
'result': f'Successfully deleted {num_deleted} expired jobs.'
}
except Exception as e:
except Exception as exception:
return {
'result': f'There was an error: {str(e)}'
'result': f'There was an error: {str(exception)}'
}
"""
Module that describes and handles the requests concerned with recording the downloads
"""
from flask import abort, request
from flask_restplus import Namespace, Resource, fields
from app.namespaces.job_statistics import record_statistics_service
from app.authorisation.decorators import token_required_for_job_id
# pylint: disable=no-self-use,redefined-builtin,invalid-name
API = Namespace('record/download', description='Requests to record statistics of a download')
DOWNLOAD_RECORD = API.model('DownloadRecord', {
......@@ -17,6 +22,7 @@ FULL_STATISTICS = API.inherit('FullDownloadRecord', DOWNLOAD_RECORD, {
'request_date': fields.Float(required=True, description='The time (POSIX timestamp) at what the job started', min=0)
})
@API.route('/<id>')
@API.param('id', 'The job identifier')
@API.response(404, 'Job not found')
......@@ -45,4 +51,4 @@ class DownloadRecord(Resource):
except record_statistics_service.JobNotFoundError:
abort(404)
except record_statistics_service.JobNotFinishedError:
abort(412)
\ No newline at end of file
abort(412)
"""
Module that describes and handles the requests concerned with recording the search events
"""
from flask import abort, request
from flask_restplus import Namespace, Resource, fields
from app.namespaces.job_statistics import record_statistics_service
from app.authorisation.decorators import token_required_for_job_id
from app.namespaces.models import delayed_job_models
# pylint: disable=redefined-builtin,invalid-name,no-self-use
API = Namespace('record/search', description='Requests to record statistics of a search')
SEARCH_RECORD = API.model('SearchRecord', {
......@@ -14,7 +19,7 @@ SEARCH_RECORD = API.model('SearchRecord', {
FULL_STATISTICS = API.inherit('FullSearchRecord', SEARCH_RECORD, {
'time_taken': fields.Integer(required=True, description='The time the job took to finish', min=0),
'search_type': fields.String(required=True, description='The type of the job ',
enum=[str(possible_type) for possible_type in delayed_job_models.JobTypes]),
enum=[str(possible_type) for possible_type in delayed_job_models.JobTypes]),
'request_date': fields.Float(required=True, description='The time (POSIX timestamp) at what the job started', min=0)
})
......
"""
Module that describes and handles the requests concerned with recording statistics
"""
from app.namespaces.models import delayed_job_models
from app.config import RUN_CONFIG
from app.es_connection import es
from app.es_connection import ES
class JobNotFoundError(Exception):
"""Base class for exceptions."""
pass
class JobNotFinishedError(Exception):
"""Base class for exceptions."""
pass
def save_statistics_for_job(id, statistics):
def save_statistics_for_job(job_id, statistics):
"""
Saves statistics (Remember to rethink this)
:param job_id:
:param statistics:
:return:
"""
try:
job = delayed_job_models.get_job_by_id(id)
job = delayed_job_models.get_job_by_id(job_id)
if job.status != delayed_job_models.JobStatuses.FINISHED:
raise JobNotFinishedError()
......@@ -28,6 +35,12 @@ def save_statistics_for_job(id, statistics):
def calculate_extra_and_save_statistics_to_elasticsearch(job, statistics):
"""
:param job:
:param statistics:
:return:
"""
calculated_statistics = {
**statistics,
......@@ -42,6 +55,11 @@ def calculate_extra_and_save_statistics_to_elasticsearch(job, statistics):
def save_search_record_to_elasticsearch(calculated_statistics):
"""
:param calculated_statistics:
:return:
"""
es_index = 'chembl_glados_es_search_record'
es_doc = {
......@@ -58,5 +76,4 @@ def save_search_record_to_elasticsearch(calculated_statistics):
print('---------------------------------------------------')
else:
print('SAVING TO ES')
res = es.search(index=es_index, body={"query": {"match_all": {}}})
ES.search(index=es_index, body={"query": {"match_all": {}}})
......@@ -2,27 +2,33 @@
Tests for the search statistics API
"""
import unittest
import datetime
import json
from app import create_app
from app.db import db
from app.db import DB
from app.namespaces.models import delayed_job_models
from app.authorisation import token_generator
import datetime
import json
# pylint: disable=too-many-locals,no-member
class TestStatus(unittest.TestCase):
"""
jljl
"""
def setUp(self):
self.flask_app = create_app()
self.client = self.flask_app.test_client()
def tearDown(self):
with self.flask_app.app_context():
delayed_job_models.delete_all_jobs()
def test_cannot_save_statistics_with_invalid_token(self):
"""
:return:
"""
with self.flask_app.app_context():
statistics = {
'total_items': 100,
......@@ -63,9 +69,11 @@ class TestStatus(unittest.TestCase):
msg='I should not be able to save statistics for a job that does not exist')
def test_job_must_be_finished_to_save_statistics(self):
"""
hkhkhkhk
:return:
"""
with self.flask_app.app_context():
job_type = delayed_job_models.JobTypes.SIMILARITY
params = {
'search_type': str(delayed_job_models.JobTypes.SIMILARITY),
......@@ -74,7 +82,6 @@ class TestStatus(unittest.TestCase):
}
with self.flask_app.app_context():
job_must_be = delayed_job_models.get_or_create(job_type, params)
statistics = {
......@@ -92,9 +99,11 @@ class TestStatus(unittest.TestCase):
self.assertEqual(response.status_code, 412, msg='The job must be finished before saving statistics')
def test_saves_statistics_for_search_job(self):
"""
hkhjk
:return: