Commit 358aa2db authored by David Mendez's avatar David Mendez
Browse files

Make pylint happy

parent f3e68cb0
......@@ -3,7 +3,7 @@
"""
from flask import Blueprint, jsonify, abort, request
from app.request_validation.decorators import validate_form_with, validate_url_params_with
from app.request_validation.decorators import validate_form_with
from app.blueprints.entities_join.controllers import marshmallow_schemas
from app.blueprints.entities_join.services import entities_join_service
from app import app_logging
......@@ -36,8 +36,8 @@ def get_link_to_related_items():
try:
json_response = entities_join_service.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description, previous_hash)
entity_from, entity_to, es_query,
selection_description, previous_hash)
return jsonify(json_response)
except entities_join_service.EntitiesJoinServiceError as error:
app_logging.debug(str(error))
......
......@@ -10,7 +10,6 @@ from app import app_logging
from app.http_cache import http_cache_utils
from utils import request_parameters
ES_PROXY_BLUEPRINT = Blueprint('es_proxy', __name__)
......@@ -66,4 +65,3 @@ def get_es_doc(index_name, doc_id):
abort(500, msg=f'Internal server error: {str(error)}')
except es_proxy_service.ESDataNotFoundError as error:
abort(404)
......@@ -20,40 +20,87 @@ class EntitiesJoinerError(Exception):
def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, raw_es_query,
raw_origin_entity, raw_destination_entity, raw_es_query,
raw_selection_description, previous_hash):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param entity_from: source entity of the items
:param entity_to: destination entity of the join
:param raw_origin_entity: text with the origin entity of the items
:param raw_destination_entity: text with the destination entity of the join
:param raw_es_query: stringifyed query in elasticsearch for the dataset
:param raw_selection_description: stringifyed javascript object describing de selection of items in the dataset
:param previous_hash: hash of the state that originated this join of entities
:return: the hash to the link with the generated state
"""
cache_key = get_cache_key(destination_entity_browser_state_template, entity_from, entity_to, raw_es_query,
raw_selection_description, previous_hash)
cache_key = get_cache_key(destination_entity_browser_state_template, raw_origin_entity, raw_destination_entity,
raw_es_query, raw_selection_description, previous_hash)
app_logging.debug(f'entities join cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
save_statistics(entity_from, entity_to, None, True)
save_statistics(raw_origin_entity, raw_destination_entity, None, True)
return cache_response
parsed_origin_entity = None
origin_destination = parse_origin_destination_parameters(raw_origin_entity, raw_destination_entity)
join_parameters = parse_join_parameters(origin_destination['parsed_origin_entity'],
origin_destination['parsed_destination_entity'],
raw_selection_description)
es_query = json.loads(raw_es_query)
ids = ids_loader.load_ids_for_query(es_query, join_parameters['selection_description'],
join_parameters['origin_property'], join_parameters['index_name'])
query_parameters = parse_query_parameters(origin_destination['parsed_origin_entity'],
origin_destination['parsed_destination_entity'])
join_state_hash = get_join_state_hash(ids, query_parameters['destination_query_generator'],
query_parameters['settings_path'], query_parameters['browser_name'],
destination_entity_browser_state_template,
previous_hash)
cache.fail_proof_set(key=cache_key, value=join_state_hash, timeout=RUN_CONFIG.get('entities_join_cache_seconds'))
save_statistics(raw_origin_entity, raw_destination_entity, len(ids), False)
return join_state_hash
def parse_origin_destination_parameters(raw_origin_entity, raw_destination_entity):
"""
:param raw_origin_entity: text with the origin entity of the items
:param raw_destination_entity: text with the destination entity of the join
:return: A dict with the parsed values of the origin and destination entities of the join
"""
if raw_destination_entity == raw_origin_entity:
raise EntitiesJoinerError(
f'entity_to ({raw_destination_entity}) and entity_from ({raw_origin_entity}) cannot be the same!')
origin_destination = {}
try:
parsed_origin_entity = standardisation.PossibleOriginEntities(entity_from)
parsed_origin_entity = standardisation.PossibleOriginEntities(raw_origin_entity)
origin_destination['parsed_origin_entity'] = parsed_origin_entity
except ValueError as error:
raise EntitiesJoinerError(
f'entity_from: {str(error)}. Possible values are {[item.value for item in standardisation.PossibleOriginEntities]}')
f'entity_from: {str(error)}. Possible values are '
f'{[item.value for item in standardisation.PossibleOriginEntities]}')
parsed_destination_entity = None
try:
parsed_destination_entity = standardisation.PossibleDestinationEntities(entity_to)
parsed_destination_entity = standardisation.PossibleDestinationEntities(raw_destination_entity)
origin_destination['parsed_destination_entity'] = parsed_destination_entity
except ValueError as error:
raise EntitiesJoinerError(
f'entity_to: {str(error)}. Possible values are {[item.value for item in standardisation.PossibleDestinationEntities]}')
f'entity_to: {str(error)}. Possible values are '
f'{[item.value for item in standardisation.PossibleDestinationEntities]}')
return origin_destination
def parse_selection_description_dict(raw_selection_description):
"""
parses the selection description from the text entered, fails if invalid
:param raw_selection_description: stringifyed javascript object describing de selection of items in the dataset
:return: a dict with the selection description entered
"""
selection_description_dict = json.loads(raw_selection_description)
try:
parsed_selection_mode = standardisation.SelectionModes(selection_description_dict['selectionMode'])
......@@ -63,39 +110,69 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
f'When selection mode is {parsed_selection_mode} there must be at least one exception')
except ValueError as error:
raise EntitiesJoinerError(
f'selectionMode: {str(error)}. Possible values are {[item.value for item in standardisation.SelectionModes]}')
f'selectionMode: {str(error)}. '
f'Possible values are {[item.value for item in standardisation.SelectionModes]}'
)
return selection_description_dict
if entity_to == entity_from:
raise EntitiesJoinerError(f'entity_to ({entity_to}) and entity_from ({entity_from}) cannot be the same!')
def parse_join_parameters(parsed_origin_entity, parsed_destination_entity, raw_selection_description):
"""
Parses the parameters for the join, fails if they are invalid
:param parsed_origin_entity: origin entity of the join
:param parsed_destination_entity: destination entity of the join
:param raw_selection_description: stringifyied javascript object describing the selection
:return: a dict with the parsed parameters
"""
join_parameters = {}
index_name = standardisation.get_index_name_for_origin_entity(parsed_origin_entity)
fail_if_null(index_name, 'index name', parsed_origin_entity, parsed_destination_entity)
fail_if_null(index_name, 'index name', parsed_origin_entity,
parsed_destination_entity)
join_parameters['index_name'] = index_name
origin_property = standardisation.get_origin_property(parsed_origin_entity, parsed_destination_entity)
fail_if_null(origin_property, 'origin property', parsed_origin_entity, parsed_destination_entity)
origin_property = standardisation.get_origin_property(parsed_origin_entity,
parsed_destination_entity)
fail_if_null(origin_property, 'origin property', parsed_origin_entity,
'parsed_destination_entity')
selection_description = json.loads(raw_selection_description)
es_query = json.loads(raw_es_query)
ids = ids_loader.load_ids_for_query(es_query, selection_description, origin_property, index_name)
join_parameters['origin_property'] = origin_property
selection_description = parse_selection_description_dict(raw_selection_description)
join_parameters['selection_description'] = selection_description
destination_query_generator = standardisation.get_destination_query_generator(parsed_origin_entity,
parsed_destination_entity)
return join_parameters
def parse_query_parameters(parsed_origin_entity, parsed_destination_entity):
"""
parses que parameters to build the join query
:param parsed_origin_entity: origin entity of the join
:param parsed_destination_entity: destination entity of the join
:return: a dict with the parsed parameters
"""
query_parameters = {}
destination_query_generator = standardisation.get_destination_query_generator(
parsed_origin_entity,
parsed_destination_entity)
fail_if_null(destination_query_generator, 'destination query generator', parsed_origin_entity,
parsed_destination_entity)
query_parameters['destination_query_generator'] = destination_query_generator
settings_path = standardisation.get_settings_path_for_destination_entity(parsed_destination_entity)
fail_if_null(settings_path, 'settings path', parsed_origin_entity, parsed_destination_entity)
settings_path = standardisation.get_settings_path_for_destination_entity(
parsed_destination_entity)
fail_if_null(settings_path, 'settings path', parsed_origin_entity,
parsed_destination_entity)
query_parameters['settings_path'] = settings_path
browser_name = standardisation.get_browser_names_for_destination_entity(parsed_destination_entity)
fail_if_null(browser_name, 'browser name', parsed_origin_entity, parsed_destination_entity)
browser_name = standardisation.get_browser_names_for_destination_entity(
parsed_destination_entity)
fail_if_null(browser_name, 'browser name', parsed_origin_entity,
parsed_destination_entity)
join_state_hash = get_join_state_hash(ids, destination_query_generator, settings_path, browser_name,
destination_entity_browser_state_template,
previous_hash)
query_parameters['browser_name'] = browser_name
cache.fail_proof_set(key=cache_key, value=join_state_hash, timeout=RUN_CONFIG.get('entities_join_cache_seconds'))
save_statistics(entity_from, entity_to, len(ids), False)
return join_state_hash
return query_parameters
def get_cache_key(destination_entity_browser_state_template,
......
"""
Module with functions that help to load the ids for the entities joiner
"""
import json
from app.entities_joiner import standardisation
from app.es_data import es_data
from utils import dict_property_access
......@@ -40,8 +38,7 @@ def get_ids_query(es_query, selection_description, from_property):
if parsed_selection_mode == standardisation.SelectionModes.ALL_ITEMS_EXCEPT:
if len(exceptions) == 0:
return get_ids_query_for_all_items(es_query, from_property)
else:
return get_ids_query_for_all_items_except_some(es_query, from_property, exceptions)
return get_ids_query_for_all_items_except_some(es_query, from_property, exceptions)
# Selecting none except some
return get_ids_query_for_no_items_except_some(es_query, from_property, exceptions)
......
......@@ -23,11 +23,12 @@ class TestEntitiesJoiner(unittest.TestCase):
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description = '{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", ' \
'"CHEMBL64"]} '
previous_hash = 'abc'
with self.assertRaises(entities_joiner.EntitiesJoinerError):
entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description)
selection_description, previous_hash)
def test_fails_when_from_is_not_on_the_possibilities(self):
"""
......@@ -42,11 +43,12 @@ class TestEntitiesJoiner(unittest.TestCase):
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description = '{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", ' \
'"CHEMBL64"]} '
previous_hash = 'abc'
with self.assertRaises(entities_joiner.EntitiesJoinerError):
entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description)
selection_description, previous_hash)
def test_fails_when_to_is_not_on_the_possibilities(self):
"""
......@@ -61,11 +63,12 @@ class TestEntitiesJoiner(unittest.TestCase):
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description = '{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", ' \
'"CHEMBL64"]} '
previous_hash = 'abc'
with self.assertRaises(entities_joiner.EntitiesJoinerError):
entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description)
selection_description, previous_hash)
def test_fails_when_selection_mode_not_in_possibilities(self):
"""
......@@ -80,11 +83,12 @@ class TestEntitiesJoiner(unittest.TestCase):
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description = '{"selectionMode": "non_existent","exceptions": ["CHEMBL3989861", "CHEMBL3989724", ' \
'"CHEMBL64"]} '
previous_hash = 'abc'
with self.assertRaises(entities_joiner.EntitiesJoinerError):
entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description)
selection_description, previous_hash)
def test_fails_when_selecting_no_items(self):
"""
......@@ -97,8 +101,9 @@ class TestEntitiesJoiner(unittest.TestCase):
es_query = '{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, ' \
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description = '{"selectionMode": "noItemsExcept","exceptions": []} '
previous_hash = 'abc'
with self.assertRaises(entities_joiner.EntitiesJoinerError):
entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description)
selection_description, previous_hash)
......@@ -12,7 +12,9 @@ from specific_tests import fun_test_simple_query, fun_test_query_with_context, \
fun_test_go_slim_target_classification, fun_test_in_vivo_assay_classification, \
fun_test_organism_taxonomy_target_classification, fun_test_protein_target_classification, \
fun_test_covid_entities_records, fun_test_database_summary, fun_test_entities_records, \
fun_test_get_all_properties, fun_test_identify_separator, fun_test_entities_join_all_tests
fun_test_get_all_properties, fun_test_identify_separator
from specific_tests.entities_join import fun_test_entities_join_all_tests
PARSER = argparse.ArgumentParser()
PARSER.add_argument('server_base_path', help='server base path to run the tests against',
......@@ -38,6 +40,7 @@ def run():
fun_test_organism_taxonomy_target_classification, fun_test_protein_target_classification,
fun_test_covid_entities_records, fun_test_database_summary, fun_test_entities_records,
fun_test_get_all_properties, fun_test_identify_separator, fun_test_entities_join_all_tests]:
test_module.run_test(ARGS.server_base_path, ARGS.delayed_jobs_server_base_path)
......
# pylint: disable=import-error
"""
Module that runs all the tests related to the entities join
"""
......@@ -189,6 +190,8 @@ def run_test(server_base_url, delayed_jobs_server_base_path):
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print('Starting entities join tests...')
print(delayed_jobs_server_base_path)
for test_description in TESTS_TO_RUN:
dataset_query = utils.load_json_data(test_description['dataset_query_path'])
selection_description = test_description['selection_description']
......
# pylint: disable=import-error
"""
Module with utils for entities join tests
"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment