Commit 7bd1e2ca authored by David Mendez's avatar David Mendez
Browse files

Add functionality for a simple es query

parent 1af5e909
......@@ -8,6 +8,7 @@ from flask_cors import CORS
from app.config import RUN_CONFIG
from app.cache import CACHE
from app.blueprints.swagger_description.swagger_description_blueprint import SWAGGER_BLUEPRINT
from app.blueprints.es_proxy.controllers.es_proxy_controller import ES_PROXY_BLUEPRINT
def create_app():
......@@ -27,6 +28,7 @@ def create_app():
flask_app.register_blueprint(SWAGGER_BLUEPRINT, url_prefix=f'{base_path}/swagger')
flask_app.register_blueprint(ES_PROXY_BLUEPRINT, url_prefix=f'{base_path}/es_data')
return flask_app
The blueprint used for handling requests to get generic es_data
from flask import Blueprint, jsonify, abort, request, send_file
from app.request_validation.decorators import validate_form_with
from app.blueprints.es_proxy.controllers import marshmallow_schemas
from import es_proxy_service
from app import app_logging
ES_PROXY_BLUEPRINT = Blueprint('es_proxy', __name__)
@ES_PROXY_BLUEPRINT.route('/get_es_data', methods = ['POST'])
def get_es_data():
:return: the json response with the data from elasticsearch
form_data = request.form
index_name = sanitise_parameter(form_data.get('index_name'))
raw_es_query = sanitise_parameter(form_data.get('es_query'))
raw_context = sanitise_parameter(form_data.get('context_obj'))
id_property = sanitise_parameter(form_data.get('id_property'))
raw_contextual_sort_data = sanitise_parameter(form_data.get('contextual_sort_data'))
app_logging.debug(f'index_name: {index_name}')
app_logging.debug(f'raw_es_query: {raw_es_query}')
app_logging.debug(f'raw_context: {raw_context}')
app_logging.debug(f'id_property: {id_property}')
app_logging.debug(f'raw_contextual_sort_data: {raw_contextual_sort_data}')
json_response = es_proxy_service.get_es_data(
return jsonify(json_response)
except es_proxy_service.ESProxyServiceError as error:
abort(500, msg=f'Internal server error: {str(error)}')
def sanitise_parameter(param_value):
Makes the parameter null if it is 'null' or 'undefined', in some cases javascript produces those values
:param param_value: value of the parameter
:return: null if 'null' or 'undefined' the actual value otherwise
if param_value == 'null' or param_value == 'undefined':
return None
return param_value
import traceback
import json
import hashlib
import base64
from django.http import JsonResponse, HttpResponse
from glados.usage_statistics import glados_server_statistics
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from django.core.cache import cache
from django.conf import settings
from import es_proxy_service
def get_es_data(request):
index_name = request.POST.get('index_name', '')
raw_search_data = request.POST.get('search_data', '')
raw_context = request.POST.get('context_obj')
id_property = request.POST.get('id_property')
raw_contextual_sort_data = request.POST.get('contextual_sort_data')
cache_key = get_request_cache_key(index_name, raw_search_data, raw_context, id_property, raw_contextual_sort_data)
cache_response = cache.get(cache_key)
if cache_response is not None:
return JsonResponse(cache_response)
if raw_context is None or raw_context == 'undefined' or raw_context == 'null':
response = glados_server_statistics.get_and_record_es_cached_response(index_name, raw_search_data)
response = es_proxy_service.get_items_with_context(index_name, raw_search_data, raw_context, id_property,
except Exception as e:
return HttpResponse('Internal Server Error', status=500)
if response is None:
return HttpResponse('ELASTIC SEARCH RESPONSE IS EMPTY!', status=500)
cache_time = settings.ES_PROXY_CACHE_SECONDS
cache.set(cache_key, response, cache_time)
return JsonResponse(response)
def get_request_cache_key(index_name, raw_search_data, raw_context, id_property, raw_contextual_sort_data):
Returns a cache key from the request parameters
:param index_name: name of the index for which the request is made
:param raw_search_data: stringified dict with the query to send to ES
:param raw_context: stringified dict describing the context of the request
:param id_property: property used to identify the items
:param raw_contextual_sort_data: stringified dict descibing the sorting by the contextual properties
stable_raw_search_data = json.dumps(json.loads(raw_search_data), sort_keys=True)
stable_raw_context = json.dumps(json.loads(raw_context), sort_keys=True)
stable_raw_contextual_sort_data = json.dumps(json.loads(raw_contextual_sort_data), sort_keys=True)
merged_params = '{index_name}-{raw_search_data}-{raw_context}-{id_property}-{raw_contextual_sort_data}'.format(
merged_params_digest = hashlib.sha256(merged_params.encode('utf-8')).digest()
base64_search_data_hash = base64.b64encode(merged_params_digest).decode('utf-8')
return 'es_proxy-{}'.format(base64_search_data_hash)
Schemas to validate the input of job status Endpoint
from marshmallow import Schema, fields
class ESProxyQuery(Schema):
Class that the schema for getting a job status job by id
index_name = fields.String(required=True)
es_query = fields.String(required=True)
context_obj = fields.String()
id_property = fields.String()
contextual_sort_data = fields.String()
Service that handles the general requests to elasticsearch data
import json
from app import app_logging
from app.es_data import es_data
class ESProxyServiceError(Exception):
"""Base class for exceptions in this file."""
def get_es_data(index_name, raw_es_query, raw_context, id_property, raw_contextual_sort_data):
:param index_name: name of the index to query
:param raw_es_query: stringifyied version of the query to send to elasticsearch
:param raw_context: stringifyied version of a JSON object describing the context of the query
:param id_property: property that identifies every item. Required when context provided
:param raw_contextual_sort_data: description of sorting if sorting by contextual properties
:return: Returns the json response from elasticsearch
if raw_context is None:
app_logging.debug('No context detected')
es_query = json.loads(raw_es_query)
return es_data.get_es_response(index_name, es_query)
return {'msg': 'hola'}
Module that handles decorators used in request validation
from flask import request, abort
from functools import wraps
def validate_form_with(validation_schema):
def wrap(func):
def wrapped_func(*args, **kwargs):
validation_errors = validation_schema().validate(request.form)
if validation_errors:
abort(400, str(validation_errors))
return func(*args, **kwargs)
return wrapped_func
return wrap
def validate_url_params_with(validation_schema):
def wrap(func):
def wrapped_func(*args, **kwargs):
validation_errors = validation_schema().validate(kwargs)
if validation_errors:
abort(400, str(validation_errors))
return func(*args, **kwargs)
return wrapped_func
return wrap
\ No newline at end of file
......@@ -21,7 +21,7 @@ tags:
- name: Get ES Data
description: Endpoint to get data from ES
- 'ES proxy'
......@@ -49,7 +49,7 @@ paths:
in: 'formData'
description: 'property that identifies every item. Required when context provided'
type: 'string'
- name: contextual_sort_data
- name: 'contextual_sort_data'
in: 'formData'
description: 'description of sorting if sorting by contextual properties'
type: 'string'
......@@ -47,4 +47,11 @@ def run_test(server_base_url):
status_code = request.status_code
print(f'status_code: {status_code}')
response_text = request.text
assert status_code == 200, 'The request failed!'
response_json = request.json()
hits = response_json['hits']['hits']
assert len(hits) > 0, 'I should have gotten hits!'
......@@ -8,4 +8,20 @@ def get_url_for_get_es_data(server_base_url):
:param server_base_url: base url of the running server. E.g.
:return: url for the get_es_data_endpoint
return f'{server_base_url}/get_es_data'
return f'{server_base_url}/es_data/get_es_data'
def print_es_response(response_text, max_chars=200):
prints the response text passed as parameter up to max_chars
:param response_text: response to print
:param max_chars: max chars to print
too_long = len(response_text) > max_chars
if too_long:
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment