Commit 4856c288 authored by David Mendez's avatar David Mendez
Browse files

Add the is_test parameters to prevent some endpoints from saving statistics

parent 1ad6f216
......@@ -8,7 +8,6 @@ from app.blueprints.contexts.controllers import marshmallow_schemas
from app.blueprints.contexts.services import contexts_service
from app.http_cache import http_cache_utils
CONTEXTS_BLUEPRINT = Blueprint('contexts', __name__)
......
"""
Marshmallow schemas for validating the properties configuration controller
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class ContextDataRequest(Schema):
class ContextDataRequest(CommonSchema):
"""
Class with the schema for getting the data of a context
"""
......
......@@ -6,6 +6,7 @@ from flask import Blueprint, jsonify, abort, request
from app.request_validation.decorators import validate_form_with
from app.blueprints.entities_join.controllers import marshmallow_schemas
from app.blueprints.entities_join.services import entities_join_service
from app.common_marshmallow import utils as schema_utils
from app import app_logging
from utils import request_parameters
......@@ -28,18 +29,20 @@ def get_link_to_related_items():
selection_description = request_parameters.sanitise_parameter(form_data.get('selection_description'))
previous_hash = request_parameters.sanitise_parameter(form_data.get('previous_hash'))
origin_id_property = request_parameters.sanitise_parameter(form_data.get('origin_id_property'))
is_test = schema_utils.parse_boolean_param(form_data, 'is_test')
app_logging.debug(f'destination_entity_browser_state_template: {destination_entity_browser_state_template}')
app_logging.debug(f'entity_from: {entity_from}')
app_logging.debug(f'entity_to: {entity_to}')
app_logging.debug(f'es_query: {es_query}')
app_logging.debug(f'selection_description: {selection_description}')
app_logging.debug(f'is_test: {is_test}')
try:
json_response = entities_join_service.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description, previous_hash,
origin_id_property)
origin_id_property, is_test)
return jsonify(json_response)
except entities_join_service.EntitiesJoinServiceError as error:
app_logging.debug(str(error))
......
"""
Schemas to validate the input of entities join Endpoint
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class EntitiesJoinQuery(Schema):
class EntitiesJoinQuery(CommonSchema):
"""
Class that the schema doing an es join
Class that defines the schema doing an es join
"""
destination_entity_browser_state_template = fields.String(required=True)
entity_from = fields.String(required=True)
......
......@@ -10,7 +10,7 @@ class EntitiesJoinServiceError(Exception):
def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description, previous_hash, origin_id_property=None):
selection_description, previous_hash, origin_id_property=None, is_test=False):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param entity_from: source entity of the items
......@@ -20,14 +20,14 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
:param previous_hash: hash of the state that originated this join of entities
:param origin_id_property: id property to identify the origin items. If not provided, it will be equal as
from_property later
:param is_test: tells if the request is marked as form a test
:return: a dict with the tiny url to the link with the generated state
"""
try:
tiny_hash = entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
entity_from, entity_to, es_query,
selection_description, previous_hash,
origin_id_property)
origin_id_property, is_test)
return {
'tiny_hash': tiny_hash
......
......@@ -6,6 +6,7 @@ from flask import Blueprint, jsonify, abort, request
from app.request_validation.decorators import validate_form_with, validate_url_params_with
from app.blueprints.es_proxy.controllers import marshmallow_schemas
from app.blueprints.es_proxy.services import es_proxy_service
from app.common_marshmallow import utils as schema_utils
from app import app_logging
from app.http_cache import http_cache_utils
from utils import request_parameters
......@@ -25,11 +26,13 @@ def get_es_data():
raw_es_query = request_parameters.sanitise_parameter(form_data.get('es_query'))
raw_context = request_parameters.sanitise_parameter(form_data.get('context_obj'))
raw_contextual_sort_data = request_parameters.sanitise_parameter(form_data.get('contextual_sort_data'))
is_test = schema_utils.parse_boolean_param(form_data, 'is_test')
app_logging.debug(f'index_name: {index_name}')
app_logging.debug(f'raw_es_query: {raw_es_query}')
app_logging.debug(f'raw_context: {raw_context}')
app_logging.debug(f'raw_contextual_sort_data: {raw_contextual_sort_data}')
app_logging.debug(f'is_test: {is_test}')
try:
......@@ -37,7 +40,9 @@ def get_es_data():
index_name,
raw_es_query,
raw_context,
raw_contextual_sort_data)
raw_contextual_sort_data,
is_test
)
http_response = jsonify(json_response)
http_cache_utils.add_cache_headers_to_response(http_response)
......
"""
Schemas to validate the input of job status Endpoint
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class ESProxyQuery(Schema):
class ESProxyQuery(CommonSchema):
"""
Class that the schema for getting es data
"""
......@@ -14,7 +16,7 @@ class ESProxyQuery(Schema):
contextual_sort_data = fields.String()
class ESProxyDoc(Schema):
class ESProxyDoc(CommonSchema):
"""
Class that the schema for getting a document by id
"""
......
......@@ -20,19 +20,20 @@ class ESDataNotFoundError(Exception):
"""Base class for exceptions in this file."""
def get_es_data(index_name, raw_es_query, raw_context, raw_contextual_sort_data):
def get_es_data(index_name, raw_es_query, raw_context, raw_contextual_sort_data, is_test=False):
"""
:param index_name: name of the index to query
:param raw_es_query: stringifyied version of the query to send to elasticsearch
:param raw_context: stringifyied version of a JSON object describing the context of the query
:param id_property: property that identifies every item. Required when context provided
:param raw_contextual_sort_data: description of sorting if sorting by contextual properties
:param is_test: tells if the request comes from a test script
:return: Returns the json response from elasticsearch and some metadata if necessary
"""
if raw_context is None:
app_logging.debug('No context detected')
es_query = json.loads(raw_es_query)
es_response = es_data.get_es_response(index_name, es_query)
es_response = es_data.get_es_response(index_name, es_query, is_test)
response = {
'es_response': es_response,
}
......@@ -45,7 +46,7 @@ def get_es_data(index_name, raw_es_query, raw_context, raw_contextual_sort_data)
context = context_loader.get_search_by_ids_context(context_dict)
subset_index_name = context['subset_index_name']
es_query = json.loads(raw_es_query)
es_response = es_data.get_es_response(subset_index_name, es_query)
es_response = es_data.get_es_response(subset_index_name, es_query, is_test)
response = {
'es_response': es_response,
......
"""
Marshmallow schemas for validating the properties configuration controller
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class PropertyConfigRequest(Schema):
class PropertyConfigRequest(CommonSchema):
"""
Class with the schema for getting the configuration of a property
"""
......@@ -12,7 +14,7 @@ class PropertyConfigRequest(Schema):
property_id = fields.String(required=True)
class GroupConfigRequest(Schema):
class GroupConfigRequest(CommonSchema):
"""
Class with the schema for getting the configuration of a group
"""
......@@ -20,7 +22,7 @@ class GroupConfigRequest(Schema):
group_name = fields.String(required=True)
class FacetsGroupConfigRequest(Schema):
class FacetsGroupConfigRequest(CommonSchema):
"""
Class with the schema for getting the configuration of a facet group
"""
......@@ -28,13 +30,14 @@ class FacetsGroupConfigRequest(Schema):
group_name = fields.String(required=True)
class IDPropertiesRequest(Schema):
class IDPropertiesRequest(CommonSchema):
"""
Class with the schema for getting the id properties of an index
"""
index_name = fields.String(required=True)
class AllPropertiesRequest(Schema):
class AllPropertiesRequest(CommonSchema):
"""
Class with the schema for getting all the properties config of an index
"""
......
"""
Marshmallow schemas for validating the queries to the search parser
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class ParseFreeTextSearchRequest(Schema):
class ParseFreeTextSearchRequest(CommonSchema):
"""
Class with the schema for parsing a free text search
"""
......
......@@ -7,6 +7,7 @@ from app.request_validation.decorators import validate_form_with
from app.blueprints.search_parser.controllers import marshmallow_schemas
from app.blueprints.search_parser.services import search_parsing_service
from app.http_cache import http_cache_utils
from app.common_marshmallow import utils as schema_utils
SEARCH_PARSER_BLUEPRINT = Blueprint('search_parsing', __name__)
......@@ -23,9 +24,10 @@ def parse_free_text_search():
search_term = form_data.get('search_term')
es_indexes = form_data.get('es_indexes')
selected_es_index = form_data.get('selected_es_index')
is_test = schema_utils.parse_boolean_param(form_data, 'is_test')
try:
raw_response = search_parsing_service.parse_search(search_term, es_indexes, selected_es_index)
raw_response = search_parsing_service.parse_search(search_term, es_indexes, selected_es_index, is_test)
http_response = jsonify(raw_response)
http_cache_utils.add_cache_headers_to_response(http_response)
return http_response
......
......@@ -13,11 +13,12 @@ class SearchParsingServiceError(Exception):
"""
def parse_search(search_term, es_indexes, selected_es_index):
def parse_search(search_term, es_indexes, selected_es_index, is_test=False):
"""
:param search_term: Term to parse
:param es_indexes: indexes in which the search will be done, separated by a comma
:param selected_es_index: es index to focus on
:param is_test: tells if it is a test request
:return: the query to send to elasticsearch based on the search term provided
"""
start_time = time.time()
......@@ -25,6 +26,7 @@ def parse_search(search_term, es_indexes, selected_es_index):
end_time = time.time()
time_taken = end_time - start_time
statistics_saver.save_free_text_search_record(time_taken)
if not is_test:
statistics_saver.save_free_text_search_record(time_taken)
return response_dict
"""
Marshmallow schemas for validating the url shortening controller
"""
from marshmallow import Schema, fields
from marshmallow import fields
from app.common_marshmallow.common_schemas import CommonSchema
class ShortenURLRequest(Schema):
class ShortenURLRequest(CommonSchema):
"""
Class with the schema for shortening a URL
"""
long_url = fields.String(required=True)
class ExpandURLRequest(Schema):
class ExpandURLRequest(CommonSchema):
"""
Class with the schema for shortening a URL
"""
......
......@@ -7,6 +7,7 @@ from app.request_validation.decorators import validate_form_with, validate_url_p
from app.blueprints.url_shortening.controllers import marshmallow_schemas
from app.blueprints.url_shortening.services import url_shortening_service
from app.http_cache import http_cache_utils
from app.common_marshmallow import utils as schema_utils
URL_SHORTENING_BLUEPRINT = Blueprint('url_shortening', __name__)
......@@ -20,9 +21,10 @@ def shorten_url():
form_data = request.form
long_url = form_data.get('long_url')
is_test = schema_utils.parse_boolean_param(form_data, 'is_test')
try:
shortening_data = url_shortening_service.shorten_url(long_url)
shortening_data = url_shortening_service.shorten_url(long_url, is_test)
http_response = jsonify(shortening_data)
http_cache_utils.add_cache_headers_to_response(http_response)
return http_response
......
......@@ -15,13 +15,14 @@ class URLNotFoundError(Exception):
"""
def shorten_url(long_url):
def shorten_url(long_url, is_test):
"""
:param long_url:
:param is_test: tells if it is a test request
:return: an object with the hash and expiration date of the hash
"""
sortened_url_dict = url_shortener.shorten_url(long_url)
sortened_url_dict = url_shortener.shorten_url(long_url, is_test)
return sortened_url_dict
......
"""
Common Schemas shared among all blueprints
"""
from marshmallow import Schema, fields
class CommonSchema(Schema):
"""
Class that defines the common schema for all blueprints
"""
# if true, means that the request is made for testing and I should save records to the usage statistics
is_test = fields.Boolean()
"""
Utils functions for processing the schemas
"""
def parse_boolean_param(request_params, field_name):
"""
parses a boolean parameter in a request
:param request_params: the dict with the request parameters
:param field_name: name of the field to parse
:return: True if value is taken as true, False otherwise
"""
field_value = request_params.get(field_name, False)
if isinstance(field_value, bool):
return field_value
if field_value.lower() == 'true':
return True
if field_value.lower() == 'false':
return False
return False
"""
Module with functions to make joins of entities
"""
# pylint: disable=too-many-locals
import json
import base64
from datetime import datetime
......@@ -21,7 +22,7 @@ class EntitiesJoinerError(Exception):
def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
raw_origin_entity, raw_destination_entity, raw_es_query,
raw_selection_description, previous_hash, origin_id_property=None):
raw_selection_description, previous_hash, origin_id_property=None, is_test=False):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param raw_origin_entity: text with the origin entity of the items
......@@ -31,6 +32,7 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
:param previous_hash: hash of the state that originated this join of entities
:param origin_id_property: id property to identify the origin items. If not provided, it will be equal as
from_property later
:param is_test: tells if the request is marked as form a test
:return: the hash to the link with the generated state
"""
cache_key = get_cache_key(destination_entity_browser_state_template, raw_origin_entity, raw_destination_entity,
......@@ -62,7 +64,8 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
previous_hash)
cache.fail_proof_set(key=cache_key, value=join_state_hash, timeout=RUN_CONFIG.get('entities_join_cache_seconds'))
save_statistics(raw_origin_entity, raw_destination_entity, len(ids), False)
if not is_test:
save_statistics(raw_origin_entity, raw_destination_entity, len(ids), False)
return join_state_hash
......
......@@ -20,11 +20,12 @@ class ESDataNotFoundError(Exception):
"""Base class for exceptions in this file."""
def get_es_response(index_name, es_query, ignore_cache=False):
def get_es_response(index_name, es_query, ignore_cache=False, is_test=False):
"""""
:param index_name: name of the index to query against
:param es_query: dict with the query to send
:param ignore_cache: determines if cache must be ignored or not
:param is_test: tells if the request comes from a test script
:return: the dict with the response from es
"""
......@@ -40,7 +41,8 @@ def get_es_response(index_name, es_query, ignore_cache=False):
end_time = time.time()
time_taken = end_time - start_time
app_logging.debug(f'results were cached')
record_that_response_was_cached(index_name, es_query, time_taken)
if not is_test:
record_that_response_was_cached(index_name, es_query, time_taken)
return cache_response
app_logging.debug(f'results were not cached')
......@@ -52,7 +54,8 @@ def get_es_response(index_name, es_query, ignore_cache=False):
end_time = time.time()
time_taken = end_time - start_time
record_that_response_not_cached(index_name, es_query, time_taken)
if not is_test:
record_that_response_not_cached(index_name, es_query, time_taken)
except elasticsearch.exceptions.RequestError as error:
app_logging.error(f'This query caused an error: ')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment