Commit e825a2aa authored by David Mendez's avatar David Mendez
Browse files

Merge branch 'staging' into 'master'

ChEMBL 29

See merge request !71
parents 46012ec1 6207e3cb
......@@ -8,7 +8,6 @@ from app.blueprints.contexts.controllers import marshmallow_schemas
from app.blueprints.contexts.services import contexts_service
from app.http_cache import http_cache_utils
CONTEXTS_BLUEPRINT = Blueprint('contexts', __name__)
......
"""
Marshmallow schemas for validating the properties configuration controller
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class ContextDataRequest(Schema):
class ContextDataRequest(CommonSchema):
"""
Class with the schema for getting the data of a context
"""
......
......@@ -6,6 +6,7 @@ from flask import Blueprint, jsonify, abort, request
from app.request_validation.decorators import validate_form_with
from app.blueprints.entities_join.controllers import marshmallow_schemas
from app.blueprints.entities_join.services import entities_join_service
from app.common_marshmallow import utils as schema_utils
from app import app_logging
from utils import request_parameters
......@@ -27,17 +28,21 @@ def get_link_to_related_items():
es_query = request_parameters.sanitise_parameter(form_data.get('es_query'))
selection_description = request_parameters.sanitise_parameter(form_data.get('selection_description'))
previous_hash = request_parameters.sanitise_parameter(form_data.get('previous_hash'))
origin_id_property = request_parameters.sanitise_parameter(form_data.get('origin_id_property'))
is_test = schema_utils.parse_boolean_param(form_data, 'is_test')
app_logging.debug(f'destination_entity_browser_state_template: {destination_entity_browser_state_template}')
app_logging.debug(f'entity_from: {entity_from}')
app_logging.debug(f'entity_to: {entity_to}')
app_logging.debug(f'es_query: {es_query}')
app_logging.debug(f'selection_description: {selection_description}')
app_logging.debug(f'is_test: {is_test}')
try:
json_response = entities_join_service.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description, previous_hash)
selection_description, previous_hash,
origin_id_property, is_test)
return jsonify(json_response)
except entities_join_service.EntitiesJoinServiceError as error:
app_logging.debug(str(error))
......
"""
Schemas to validate the input of entities join Endpoint
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class EntitiesJoinQuery(Schema):
class EntitiesJoinQuery(CommonSchema):
"""
Class that the schema doing an es join
Class that defines the schema doing an es join
"""
destination_entity_browser_state_template = fields.String(required=True)
entity_from = fields.String(required=True)
......@@ -14,3 +16,4 @@ class EntitiesJoinQuery(Schema):
es_query = fields.String(required=True)
selection_description = fields.String(required=True)
previous_hash = fields.String()
origin_id_property = fields.String()
......@@ -10,7 +10,7 @@ class EntitiesJoinServiceError(Exception):
def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description, previous_hash):
selection_description, previous_hash, origin_id_property=None, is_test=False):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param entity_from: source entity of the items
......@@ -18,13 +18,16 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
:param es_query: query in elasticsearch for the dataset
:param selection_description: stringifyed javascript object describing de selection of items in the dataset
:param previous_hash: hash of the state that originated this join of entities
:param origin_id_property: id property to identify the origin items. If not provided, it will be equal as
from_property later
:param is_test: tells if the request is marked as form a test
:return: a dict with the tiny url to the link with the generated state
"""
try:
tiny_hash = entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description, previous_hash)
selection_description, previous_hash,
origin_id_property, is_test)
return {
'tiny_hash': tiny_hash
......
......@@ -6,6 +6,7 @@ from flask import Blueprint, jsonify, abort, request
from app.request_validation.decorators import validate_form_with, validate_url_params_with
from app.blueprints.es_proxy.controllers import marshmallow_schemas
from app.blueprints.es_proxy.services import es_proxy_service
from app.common_marshmallow import utils as schema_utils
from app import app_logging
from app.http_cache import http_cache_utils
from utils import request_parameters
......@@ -25,11 +26,13 @@ def get_es_data():
raw_es_query = request_parameters.sanitise_parameter(form_data.get('es_query'))
raw_context = request_parameters.sanitise_parameter(form_data.get('context_obj'))
raw_contextual_sort_data = request_parameters.sanitise_parameter(form_data.get('contextual_sort_data'))
is_test = schema_utils.parse_boolean_param(form_data, 'is_test')
app_logging.debug(f'index_name: {index_name}')
app_logging.debug(f'raw_es_query: {raw_es_query}')
app_logging.debug(f'raw_context: {raw_context}')
app_logging.debug(f'raw_contextual_sort_data: {raw_contextual_sort_data}')
app_logging.debug(f'is_test: {is_test}')
try:
......@@ -37,7 +40,9 @@ def get_es_data():
index_name,
raw_es_query,
raw_context,
raw_contextual_sort_data)
raw_contextual_sort_data,
is_test
)
http_response = jsonify(json_response)
http_cache_utils.add_cache_headers_to_response(http_response)
......
"""
Schemas to validate the input of job status Endpoint
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class ESProxyQuery(Schema):
class ESProxyQuery(CommonSchema):
"""
Class that the schema for getting es data
"""
......@@ -14,7 +16,7 @@ class ESProxyQuery(Schema):
contextual_sort_data = fields.String()
class ESProxyDoc(Schema):
class ESProxyDoc(CommonSchema):
"""
Class that the schema for getting a document by id
"""
......
......@@ -20,19 +20,20 @@ class ESDataNotFoundError(Exception):
"""Base class for exceptions in this file."""
def get_es_data(index_name, raw_es_query, raw_context, raw_contextual_sort_data):
def get_es_data(index_name, raw_es_query, raw_context, raw_contextual_sort_data, is_test=False):
"""
:param index_name: name of the index to query
:param raw_es_query: stringifyied version of the query to send to elasticsearch
:param raw_context: stringifyied version of a JSON object describing the context of the query
:param id_property: property that identifies every item. Required when context provided
:param raw_contextual_sort_data: description of sorting if sorting by contextual properties
:param is_test: tells if the request comes from a test script
:return: Returns the json response from elasticsearch and some metadata if necessary
"""
if raw_context is None:
app_logging.debug('No context detected')
es_query = json.loads(raw_es_query)
es_response = es_data.get_es_response(index_name, es_query)
es_response = es_data.get_es_response(index_name, es_query, is_test)
response = {
'es_response': es_response,
}
......@@ -45,7 +46,7 @@ def get_es_data(index_name, raw_es_query, raw_context, raw_contextual_sort_data)
context = context_loader.get_search_by_ids_context(context_dict)
subset_index_name = context['subset_index_name']
es_query = json.loads(raw_es_query)
es_response = es_data.get_es_response(subset_index_name, es_query)
es_response = es_data.get_es_response(subset_index_name, es_query, is_test)
response = {
'es_response': es_response,
......
"""
Marshmallow schemas for validating the properties configuration controller
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class PropertyConfigRequest(Schema):
class PropertyConfigRequest(CommonSchema):
"""
Class with the schema for getting the configuration of a property
"""
......@@ -12,7 +14,7 @@ class PropertyConfigRequest(Schema):
property_id = fields.String(required=True)
class GroupConfigRequest(Schema):
class GroupConfigRequest(CommonSchema):
"""
Class with the schema for getting the configuration of a group
"""
......@@ -20,7 +22,7 @@ class GroupConfigRequest(Schema):
group_name = fields.String(required=True)
class FacetsGroupConfigRequest(Schema):
class FacetsGroupConfigRequest(CommonSchema):
"""
Class with the schema for getting the configuration of a facet group
"""
......@@ -28,13 +30,14 @@ class FacetsGroupConfigRequest(Schema):
group_name = fields.String(required=True)
class IDPropertiesRequest(Schema):
class IDPropertiesRequest(CommonSchema):
"""
Class with the schema for getting the id properties of an index
"""
index_name = fields.String(required=True)
class AllPropertiesRequest(Schema):
class AllPropertiesRequest(CommonSchema):
"""
Class with the schema for getting all the properties config of an index
"""
......
"""
Marshmallow schemas for validating the queries to the search parser
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class ParseFreeTextSearchRequest(Schema):
class ParseFreeTextSearchRequest(CommonSchema):
"""
Class with the schema for parsing a free text search
"""
......
......@@ -7,6 +7,7 @@ from app.request_validation.decorators import validate_form_with
from app.blueprints.search_parser.controllers import marshmallow_schemas
from app.blueprints.search_parser.services import search_parsing_service
from app.http_cache import http_cache_utils
from app.common_marshmallow import utils as schema_utils
SEARCH_PARSER_BLUEPRINT = Blueprint('search_parsing', __name__)
......@@ -23,9 +24,10 @@ def parse_free_text_search():
search_term = form_data.get('search_term')
es_indexes = form_data.get('es_indexes')
selected_es_index = form_data.get('selected_es_index')
is_test = schema_utils.parse_boolean_param(form_data, 'is_test')
try:
raw_response = search_parsing_service.parse_search(search_term, es_indexes, selected_es_index)
raw_response = search_parsing_service.parse_search(search_term, es_indexes, selected_es_index, is_test)
http_response = jsonify(raw_response)
http_cache_utils.add_cache_headers_to_response(http_response)
return http_response
......
......@@ -13,11 +13,12 @@ class SearchParsingServiceError(Exception):
"""
def parse_search(search_term, es_indexes, selected_es_index):
def parse_search(search_term, es_indexes, selected_es_index, is_test=False):
"""
:param search_term: Term to parse
:param es_indexes: indexes in which the search will be done, separated by a comma
:param selected_es_index: es index to focus on
:param is_test: tells if it is a test request
:return: the query to send to elasticsearch based on the search term provided
"""
start_time = time.time()
......@@ -25,6 +26,7 @@ def parse_search(search_term, es_indexes, selected_es_index):
end_time = time.time()
time_taken = end_time - start_time
statistics_saver.save_free_text_search_record(time_taken)
if not is_test:
statistics_saver.save_free_text_search_record(time_taken)
return response_dict
"""
Marshmallow schemas for validating the url shortening controller
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class ShortenURLRequest(Schema):
class ShortenURLRequest(CommonSchema):
"""
Class with the schema for shortening a URL
"""
long_url = fields.String(required=True)
class ExpandURLRequest(Schema):
class ExpandURLRequest(CommonSchema):
"""
Class with the schema for shortening a URL
"""
......
......@@ -7,6 +7,7 @@ from app.request_validation.decorators import validate_form_with, validate_url_p
from app.blueprints.url_shortening.controllers import marshmallow_schemas
from app.blueprints.url_shortening.services import url_shortening_service
from app.http_cache import http_cache_utils
from app.common_marshmallow import utils as schema_utils
URL_SHORTENING_BLUEPRINT = Blueprint('url_shortening', __name__)
......@@ -20,9 +21,10 @@ def shorten_url():
form_data = request.form
long_url = form_data.get('long_url')
is_test = schema_utils.parse_boolean_param(form_data, 'is_test')
try:
shortening_data = url_shortening_service.shorten_url(long_url)
shortening_data = url_shortening_service.shorten_url(long_url, is_test)
http_response = jsonify(shortening_data)
http_cache_utils.add_cache_headers_to_response(http_response)
return http_response
......
......@@ -15,13 +15,14 @@ class URLNotFoundError(Exception):
"""
def shorten_url(long_url):
def shorten_url(long_url, is_test):
"""
:param long_url:
:param is_test: tells if it is a test request
:return: an object with the hash and expiration date of the hash
"""
sortened_url_dict = url_shortener.shorten_url(long_url)
sortened_url_dict = url_shortener.shorten_url(long_url, is_test)
return sortened_url_dict
......
......@@ -109,7 +109,8 @@ def entities_records():
'Cells': get_entity_total_count('chembl_cell_line'),
'Tissues': get_entity_total_count('chembl_tissue'),
'Indications': get_entity_total_count('chembl_drug_indication_by_parent'),
'Mechanisms': get_entity_total_count('chembl_mechanism_by_parent_target')
'Mechanisms': get_entity_total_count('chembl_mechanism_by_parent_target'),
'Drug Warnings': get_entity_total_count('chembl_drug_warning_by_parent')
}
......
"""
Common Schemas shared among all blueprints
"""
from marshmallow import Schema, fields
class CommonSchema(Schema):
"""
Class that defines the common schema for all blueprints
"""
# if true, means that the request is made for testing and I should save records to the usage statistics
is_test = fields.Boolean()
"""
Utils functions for processing the schemas
"""
def parse_boolean_param(request_params, field_name):
"""
parses a boolean parameter in a request
:param request_params: the dict with the request parameters
:param field_name: name of the field to parse
:return: True if value is taken as true, False otherwise
"""
field_value = request_params.get(field_name, False)
if isinstance(field_value, bool):
return field_value
if field_value.lower() == 'true':
return True
if field_value.lower() == 'false':
return False
return False
"""
Module with functions to make joins of entities
"""
# pylint: disable=too-many-locals
import json
import base64
from datetime import datetime
......@@ -21,7 +22,7 @@ class EntitiesJoinerError(Exception):
def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
raw_origin_entity, raw_destination_entity, raw_es_query,
raw_selection_description, previous_hash):
raw_selection_description, previous_hash, origin_id_property=None, is_test=False):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param raw_origin_entity: text with the origin entity of the items
......@@ -29,6 +30,9 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
:param raw_es_query: stringifyed query in elasticsearch for the dataset
:param raw_selection_description: stringifyed javascript object describing de selection of items in the dataset
:param previous_hash: hash of the state that originated this join of entities
:param origin_id_property: id property to identify the origin items. If not provided, it will be equal as
from_property later
:param is_test: tells if the request is marked as form a test
:return: the hash to the link with the generated state
"""
cache_key = get_cache_key(destination_entity_browser_state_template, raw_origin_entity, raw_destination_entity,
......@@ -48,7 +52,8 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
es_query = json.loads(raw_es_query)
ids = ids_loader.load_ids_for_query(es_query, join_parameters['selection_description'],
join_parameters['origin_property'], join_parameters['index_name'])
join_parameters['origin_property'], join_parameters['index_name'],
origin_id_property)
query_parameters = parse_query_parameters(origin_destination['parsed_origin_entity'],
origin_destination['parsed_destination_entity'])
......@@ -56,10 +61,11 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
join_state_hash = get_join_state_hash(ids, query_parameters['destination_query_generator'],
query_parameters['settings_path'], query_parameters['browser_name'],
destination_entity_browser_state_template,
previous_hash)
previous_hash, is_test)
cache.fail_proof_set(key=cache_key, value=join_state_hash, timeout=RUN_CONFIG.get('entities_join_cache_seconds'))
save_statistics(raw_origin_entity, raw_destination_entity, len(ids), False)
if not is_test:
save_statistics(raw_origin_entity, raw_destination_entity, len(ids), False)
return join_state_hash
......@@ -210,7 +216,7 @@ def fail_if_null(value, value_name, parsed_origin_entity, parsed_destination_ent
def get_join_state_hash(ids, destination_query_generator, settings_path, browser_name,
destination_entity_browser_state_template,
previous_hash):
previous_hash, is_test=False):
"""
:param ids: list of its for the join
:param destination_query_generator: function that generates the join query from a list of ids
......@@ -218,6 +224,7 @@ def get_join_state_hash(ids, destination_query_generator, settings_path, browser
:param browser_name: name of the destination browser used to build the desired url
:param destination_entity_browser_state_template: template to build the url for the destination entity browser
:param previous_hash: hash of the state that originated this join of entities
:param is_test: indicates if it is a test request
:return: the hash for the query state used for the join
"""
desired_state = {
......@@ -244,15 +251,16 @@ def get_join_state_hash(ids, destination_query_generator, settings_path, browser
)
hashable_part = f'#{destination_entity_browser_state_url.split("#")[1]}'
return get_destination_url_hash(hashable_part)
return get_destination_url_hash(hashable_part, is_test)
def get_destination_url_hash(hashable_part):
def get_destination_url_hash(hashable_part, is_test=False):
"""
:param hashable_part: of the destination url generated
:param is_test: indicates if it is a test request
:return: a hash generated using the hasable url part of the destination browser
"""
shortening_response = url_shortener.shorten_url(hashable_part)
shortening_response = url_shortener.shorten_url(hashable_part, is_test)
return shortening_response['hash']
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment