Commit f71c8cb2 authored by David Mendez's avatar David Mendez
Browse files

Entities join: cache response

parent e4796ea1
......@@ -40,8 +40,4 @@ def get_link_to_related_items():
selection_description, previous_hash)
return jsonify(json_response)
except entities_join_service.EntitiesJoinServiceError as error:
print('error!')
print(error)
abort(500, f'Internal server error: {str(error)}')
......@@ -4,12 +4,15 @@ Module with functions to make joins of entities
import json
import base64
from datetime import datetime
import hashlib
from app.entities_joiner import standardisation
from app.entities_joiner import ids_loader
from app.url_shortening import url_shortener
from app.config import RUN_CONFIG
from app.usage_statistics import statistics_saver
from app import app_logging
from app import cache
class EntitiesJoinerError(Exception):
......@@ -28,19 +31,28 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
:param previous_hash: hash of the state that originated this join of entities
:return: the hash to the link with the generated state
"""
parsed_from_entity = None
cache_key = get_cache_key(destination_entity_browser_state_template, entity_from, entity_to, raw_es_query,
raw_selection_description, previous_hash)
app_logging.debug(f'entities join cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is None:
save_statistics(entity_from, entity_to, None, True)
return cache_response
parsed_origin_entity = None
try:
parsed_from_entity = standardisation.PossibleEntitiesFrom(entity_from)
parsed_origin_entity = standardisation.PossibleOriginEntities(entity_from)
except ValueError as error:
raise EntitiesJoinerError(
f'entity_from: {str(error)}. Possible values are {[item.value for item in standardisation.PossibleEntitiesFrom]}')
f'entity_from: {str(error)}. Possible values are {[item.value for item in standardisation.PossibleOriginEntities]}')
parsed_to_entity = None
parsed_destination_entity = None
try:
parsed_to_entity = standardisation.PossibleEntitiesTo(entity_to)
parsed_destination_entity = standardisation.PossibleDestinationEntities(entity_to)
except ValueError as error:
raise EntitiesJoinerError(
f'entity_to: {str(error)}. Possible values are {[item.value for item in standardisation.PossibleEntitiesTo]}')
f'entity_to: {str(error)}. Possible values are {[item.value for item in standardisation.PossibleDestinationEntities]}')
selection_description_dict = json.loads(raw_selection_description)
try:
......@@ -56,44 +68,65 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
if entity_to == entity_from:
raise EntitiesJoinerError(f'entity_to ({entity_to}) and entity_from ({entity_from}) cannot be the same!')
index_name = standardisation.get_index_name_for_from_entity(parsed_from_entity)
fail_if_null(index_name, 'index name', parsed_from_entity, parsed_to_entity)
index_name = standardisation.get_index_name_for_origin_entity(parsed_origin_entity)
fail_if_null(index_name, 'index name', parsed_origin_entity, parsed_destination_entity)
from_property = standardisation.get_from_property(parsed_from_entity, parsed_to_entity)
fail_if_null(from_property, 'from property', parsed_from_entity, parsed_to_entity)
from_property = standardisation.get_from_property(parsed_origin_entity, parsed_destination_entity)
fail_if_null(from_property, 'from property', parsed_origin_entity, parsed_destination_entity)
selection_description = json.loads(raw_selection_description)
es_query = json.loads(raw_es_query)
ids = ids_loader.load_ids_for_query(es_query, selection_description, from_property, index_name)
to_property = standardisation.get_to_property(parsed_from_entity, parsed_to_entity)
fail_if_null(to_property, 'to property', parsed_from_entity, parsed_to_entity)
to_property = standardisation.get_to_property(parsed_origin_entity, parsed_destination_entity)
fail_if_null(to_property, 'to property', parsed_origin_entity, parsed_destination_entity)
settings_path = standardisation.get_settings_path_for_to_entity(parsed_to_entity)
fail_if_null(settings_path, 'settings path', parsed_from_entity, parsed_to_entity)
settings_path = standardisation.get_settings_path_for_destination_entity(parsed_destination_entity)
fail_if_null(settings_path, 'settings path', parsed_origin_entity, parsed_destination_entity)
browser_name = standardisation.get_browser_names_for_to_entity(parsed_to_entity)
fail_if_null(browser_name, 'browser name', parsed_from_entity, parsed_to_entity)
browser_name = standardisation.get_browser_names_for_destination_entity(parsed_destination_entity)
fail_if_null(browser_name, 'browser name', parsed_origin_entity, parsed_destination_entity)
join_state_hash = get_join_state_hash(ids, to_property, settings_path, browser_name,
destination_entity_browser_state_template,
previous_hash)
save_statistics(entity_from, entity_to, len(ids))
cache.fail_proof_set(key=cache_key, value=join_state_hash, timeout=RUN_CONFIG.get('entities_join_cache_seconds'))
save_statistics(entity_from, entity_to, len(ids), False)
return join_state_hash
def fail_if_null(value, value_name, parsed_from_entity, parsed_to_entity):
def get_cache_key(destination_entity_browser_state_template,
entity_from, entity_to, raw_es_query,
raw_selection_description, previous_hash):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param entity_from: source entity of the items
:param entity_to: destination entity of the join
:param raw_es_query: stringifyed query in elasticsearch for the dataset
:param raw_selection_description: stringifyed javascript object describing de selection of items in the dataset
:param previous_hash: hash of the state that originated this join of entities
:return: a cache key corresponding to the parameters given
"""
query_identifier = f'{destination_entity_browser_state_template}-{entity_from}-{entity_to}-{raw_es_query}-' \
f'{raw_selection_description}-{previous_hash}'
query_identifier_digest = hashlib.sha256(query_identifier.encode('utf-8')).digest()
base64_identifier_hash = base64.b64encode(query_identifier_digest).decode('utf-8')
return base64_identifier_hash
def fail_if_null(value, value_name, parsed_origin_entity, parsed_destination_entity):
"""
Fails if the value is null and uses the rest of the parameters to build an error message
:param value: value to check
:param value_name: name of the value for the error message
:param parsed_from_entity: parsed from entity
:param parsed_to_entity: parsed to entity
:param parsed_origin_entity: parsed from entity
:param parsed_destination_entity: parsed to entity
"""
if value is None:
raise EntitiesJoinerError(f'There is no {value_name} configured for queries from {parsed_from_entity.value} '
f'to {parsed_to_entity.value}')
raise EntitiesJoinerError(f'There is no {value_name} configured for queries from {parsed_origin_entity.value} '
f'to {parsed_destination_entity.value}')
def get_join_state_hash(ids, to_property, settings_path, browser_name, destination_entity_browser_state_template,
......@@ -144,11 +177,12 @@ def get_destination_url_hash(hashable_part):
return shortening_response['hash']
def save_statistics(raw_entity_from, raw_entity_to, num_ids):
def save_statistics(raw_entity_from, raw_entity_to, num_ids, is_cached):
"""
:param raw_entity_from: source entity of the items
:param raw_entity_to: destination entity of the join
:param num_ids: number of ids joined
:param is_cached: boolean indicating if the join is cached or not
"""
statistics_document = {
......@@ -156,6 +190,7 @@ def save_statistics(raw_entity_from, raw_entity_to, num_ids):
'entity_to': raw_entity_to,
'num_ids': num_ids,
'request_date': datetime.utcnow().timestamp() * 1000,
'is_cached': is_cached
}
index_name = RUN_CONFIG.get('usage_statistics').get('entities_join_statistics_index')
......
......@@ -4,7 +4,7 @@ Module that helps with the standardisation of the package
from enum import Enum
class PossibleEntitiesFrom(Enum):
class PossibleOriginEntities(Enum):
"""
Enumeration with the possible froms allowed
"""
......@@ -18,7 +18,7 @@ class PossibleEntitiesFrom(Enum):
CHEMBL_DRUG_WARNINGS = 'CHEMBL_DRUG_WARNINGS'
class PossibleEntitiesTo(Enum):
class PossibleDestinationEntities(Enum):
"""
Enumeration with the possible to allowed
"""
......@@ -44,48 +44,48 @@ class SelectionModes(Enum):
INDEX_NAMES_FOR_FROM_ENTITIES = {
PossibleEntitiesFrom.CHEMBL_ACTIVITIES: 'chembl_activity',
PossibleEntitiesFrom.CHEMBL_COMPOUNDS: 'chembl_molecule',
PossibleEntitiesFrom.CHEMBL_TARGETS: 'chembl_target',
PossibleEntitiesFrom.CHEMBL_ASSAYS: 'chembl_assay',
PossibleEntitiesFrom.CHEMBL_DOCUMENTS: 'chembl_document',
PossibleEntitiesFrom.CHEMBL_CELL_LINES: 'chembl_cell_line',
PossibleEntitiesFrom.CHEMBL_TISSUES: 'chembl_tissue',
PossibleEntitiesFrom.CHEMBL_DRUG_WARNINGS: 'chembl_drug_warning_by_parent',
PossibleOriginEntities.CHEMBL_ACTIVITIES: 'chembl_activity',
PossibleOriginEntities.CHEMBL_COMPOUNDS: 'chembl_molecule',
PossibleOriginEntities.CHEMBL_TARGETS: 'chembl_target',
PossibleOriginEntities.CHEMBL_ASSAYS: 'chembl_assay',
PossibleOriginEntities.CHEMBL_DOCUMENTS: 'chembl_document',
PossibleOriginEntities.CHEMBL_CELL_LINES: 'chembl_cell_line',
PossibleOriginEntities.CHEMBL_TISSUES: 'chembl_tissue',
PossibleOriginEntities.CHEMBL_DRUG_WARNINGS: 'chembl_drug_warning_by_parent',
}
def get_index_name_for_from_entity(parsed_from_entity):
def get_index_name_for_origin_entity(parsed_origin_entity):
"""
:param parsed_from_entity: entity 'from' parsed by the PossibleEntitiesFrom enum
:param parsed_origin_entity: entity 'from' parsed by the PossibleEntitiesFrom enum
:return: the index name corresponding to the from entity
"""
index_name = INDEX_NAMES_FOR_FROM_ENTITIES.get(parsed_from_entity)
index_name = INDEX_NAMES_FOR_FROM_ENTITIES.get(parsed_origin_entity)
return index_name
JOIN_PROPERTIES = {
'from': {
PossibleEntitiesFrom.CHEMBL_DRUG_WARNINGS: {
PossibleOriginEntities.CHEMBL_DRUG_WARNINGS: {
'to': {
PossibleEntitiesTo.CHEMBL_ACTIVITIES: {
PossibleDestinationEntities.CHEMBL_ACTIVITIES: {
'from_property': 'drug_warning.molecule_chembl_id',
'to_property': 'molecule_chembl_id'
},
PossibleEntitiesTo.CHEMBL_COMPOUNDS: {
PossibleDestinationEntities.CHEMBL_COMPOUNDS: {
'from_property': 'drug_warning.molecule_chembl_id',
'to_property': 'molecule_chembl_id'
},
PossibleEntitiesTo.CHEMBL_DRUGS: {
PossibleDestinationEntities.CHEMBL_DRUGS: {
'from_property': 'drug_warning.parent_molecule_chembl_id',
'to_property': 'molecule_chembl_id'
},
PossibleEntitiesTo.CHEMBL_DRUG_MECHANISMS: {
PossibleDestinationEntities.CHEMBL_DRUG_MECHANISMS: {
'from_property': 'drug_warning.parent_molecule_chembl_id',
'to_property': 'molecule_chembl_id'
},
PossibleEntitiesTo.CHEMBL_DRUG_INDICATIONS: {
PossibleDestinationEntities.CHEMBL_DRUG_INDICATIONS: {
'from_property': 'drug_warning.parent_molecule_chembl_id',
'to_property': 'molecule_chembl_id'
}
......@@ -95,60 +95,60 @@ JOIN_PROPERTIES = {
}
# This is used by the frontend to know which settings use to load the page, should be discarded in the future
SETTINGS_PATHS_FOR_TO_ENTITIES = {
PossibleEntitiesTo.CHEMBL_ACTIVITIES: 'ES_INDEXES_NO_MAIN_SEARCH.ACTIVITY',
PossibleEntitiesTo.CHEMBL_COMPOUNDS: 'ES_INDEXES_NO_MAIN_SEARCH.COMPOUND_COOL_CARDS',
PossibleEntitiesTo.CHEMBL_DRUGS: 'ES_INDEXES_NO_MAIN_SEARCH.DRUGS_LIST',
PossibleEntitiesTo.CHEMBL_DRUG_MECHANISMS: 'ES_INDEXES_NO_MAIN_SEARCH.MECHANISMS_OF_ACTION',
PossibleEntitiesTo.CHEMBL_DRUG_INDICATIONS: 'ES_INDEXES_NO_MAIN_SEARCH.DRUG_INDICATIONS'
SETTINGS_PATHS_FOR_DESTINATION_ENTITIES = {
PossibleDestinationEntities.CHEMBL_ACTIVITIES: 'ES_INDEXES_NO_MAIN_SEARCH.ACTIVITY',
PossibleDestinationEntities.CHEMBL_COMPOUNDS: 'ES_INDEXES_NO_MAIN_SEARCH.COMPOUND_COOL_CARDS',
PossibleDestinationEntities.CHEMBL_DRUGS: 'ES_INDEXES_NO_MAIN_SEARCH.DRUGS_LIST',
PossibleDestinationEntities.CHEMBL_DRUG_MECHANISMS: 'ES_INDEXES_NO_MAIN_SEARCH.MECHANISMS_OF_ACTION',
PossibleDestinationEntities.CHEMBL_DRUG_INDICATIONS: 'ES_INDEXES_NO_MAIN_SEARCH.DRUG_INDICATIONS'
}
BROWSER_NAMES_FOR_TO_ENTITIES = {
PossibleEntitiesTo.CHEMBL_ACTIVITIES: 'activities',
PossibleEntitiesTo.CHEMBL_COMPOUNDS: 'compounds',
PossibleEntitiesTo.CHEMBL_DRUGS: 'drugs',
PossibleEntitiesTo.CHEMBL_DRUG_MECHANISMS: 'mechanisms_of_action',
PossibleEntitiesTo.CHEMBL_DRUG_INDICATIONS: 'drug_indications'
BROWSER_NAMES_FOR_DESTINATION_ENTITIES = {
PossibleDestinationEntities.CHEMBL_ACTIVITIES: 'activities',
PossibleDestinationEntities.CHEMBL_COMPOUNDS: 'compounds',
PossibleDestinationEntities.CHEMBL_DRUGS: 'drugs',
PossibleDestinationEntities.CHEMBL_DRUG_MECHANISMS: 'mechanisms_of_action',
PossibleDestinationEntities.CHEMBL_DRUG_INDICATIONS: 'drug_indications'
}
def get_from_property(parsed_from_entity, parsed_to_entity):
def get_from_property(parsed_origin_entity, parsed_destination_entity):
"""
:param parsed_from_entity: entity 'from' parsed by the PossibleEntitiesFrom enum
:param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:param parsed_origin_entity: entity 'from' parsed by the PossibleEntitiesFrom enum
:param parsed_destination_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:return: the property used in the from entity of the join
"""
return JOIN_PROPERTIES. \
get('from', {}). \
get(parsed_from_entity, {}). \
get(parsed_origin_entity, {}). \
get('to', {}). \
get(parsed_to_entity, {}).get('from_property')
get(parsed_destination_entity, {}).get('from_property')
def get_to_property(parsed_from_entity, parsed_to_entity):
def get_to_property(parsed_origin_entity, parsed_destination_entity):
"""
:param parsed_from_entity: entity 'from' parsed by the PossibleEntitiesFrom enum
:param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:param parsed_origin_entity: entity 'from' parsed by the PossibleEntitiesFrom enum
:param parsed_destination_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:return: the property used in the from entity of the join
"""
return JOIN_PROPERTIES. \
get('from', {}). \
get(parsed_from_entity, {}). \
get(parsed_origin_entity, {}). \
get('to', {}). \
get(parsed_to_entity, {}).get('to_property')
get(parsed_destination_entity, {}).get('to_property')
def get_settings_path_for_to_entity(parsed_to_entity):
def get_settings_path_for_destination_entity(parsed_destination_entity):
"""
:param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:param parsed_destination_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:return: the settings path used to create the destination page
"""
return SETTINGS_PATHS_FOR_TO_ENTITIES.get(parsed_to_entity)
return SETTINGS_PATHS_FOR_DESTINATION_ENTITIES.get(parsed_destination_entity)
def get_browser_names_for_to_entity(parsed_to_entity):
def get_browser_names_for_destination_entity(parsed_destination_entity):
"""
:param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:param parsed_destination_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:return: the browser name used in the url for the destination page
"""
return BROWSER_NAMES_FOR_TO_ENTITIES.get(parsed_to_entity)
return BROWSER_NAMES_FOR_DESTINATION_ENTITIES.get(parsed_destination_entity)
......@@ -238,8 +238,6 @@ def get_es_scanner(index_name, ids_query):
:param ids_query: query to get the ids
:return: a elasticsearch scanner for the query and the parameters given
"""
print('get_es_scanner')
print('index_name: ', index_name)
return scan(
ES,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment