Commit b438b304 authored by David Mendez's avatar David Mendez
Browse files

Entities join endpoint: add more functional tests

parent 0f026994
......@@ -139,4 +139,5 @@ app/static/
output.csv
deleteme
deleteme.json
\ No newline at end of file
......@@ -2,9 +2,11 @@
Module with functions to make joins of entities
"""
import json
import base64
from app.entities_joiner import standardisation
from app.entities_joiner import ids_loader
from app.url_shortening import url_shortener
class EntitiesJoinerError(Exception):
......@@ -53,15 +55,11 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
print('load ids:')
index_name = standardisation.get_index_name_for_from_entity(parsed_from_entity)
if index_name is None:
raise EntitiesJoinerError(f'There is no index name configured for queries from {parsed_from_entity.value}')
fail_if_null(index_name, 'index name', parsed_from_entity, parsed_to_entity)
print('index_name: ', index_name)
from_property = standardisation.get_from_property(parsed_from_entity, parsed_to_entity)
if from_property is None:
raise EntitiesJoinerError(f'There is no from property configured for queries from {parsed_from_entity.value}')
fail_if_null(from_property, 'from property', parsed_from_entity, parsed_to_entity)
print('from_property: ', from_property)
selection_description = json.loads(raw_selection_description)
......@@ -69,18 +67,43 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
ids = ids_loader.load_ids_for_query(es_query, selection_description, from_property, index_name)
to_property = standardisation.get_to_property(parsed_from_entity, parsed_to_entity)
join_query = get_join_query(ids, to_property)
print('join_query: ')
print(join_query)
fail_if_null(to_property, 'to property', parsed_from_entity, parsed_to_entity)
settings_path = standardisation.get_settings_path_for_to_entity(parsed_to_entity)
fail_if_null(settings_path, 'settings path', parsed_from_entity, parsed_to_entity)
browser_name = standardisation.get_browser_names_for_to_entity(parsed_to_entity)
fail_if_null(browser_name, 'browser name', parsed_from_entity, parsed_to_entity)
join_state_hash = get_join_state_hash(ids, to_property, settings_path, browser_name,
destination_entity_browser_state_template)
print('join_state_hash: ')
print(join_state_hash)
return join_state_hash
return 'holaaaa'
def fail_if_null(value, value_name, parsed_from_entity, parsed_to_entity):
"""
Fails if the value is null and uses the rest of the parameters to build an error message
:param value: value to check
:param value_name: name of the value for the error message
:param parsed_from_entity: parsed from entity
:param parsed_to_entity: parsed to entity
"""
if value is None:
raise EntitiesJoinerError(f'There is no {value_name} configured for queries from {parsed_from_entity.value} '
f'to {parsed_to_entity.value}')
def get_join_query(ids, to_property):
def get_join_state_hash(ids, to_property, settings_path, browser_name, destination_entity_browser_state_template):
"""
:param ids: list of its for the join
:param to_property: entity to which to do the join
:return: the query to use for the join
:param settings_path: settings path to be used by the user interface
:param browser_name: name of the destination browser used to build the desired url
:param destination_entity_browser_state_template: template to build the url for the destination entity browser
:return: the hash for the query state used for the join
"""
print('get_join_query')
print('ids: ')
......@@ -88,11 +111,41 @@ def get_join_query(ids, to_property):
print('to_property: ')
print(to_property)
return
{
"query": {
"terms": {
"molecule_chembl_id": ["CHEMBL2107495", "CHEMBL1204165"]
}
ids_clauses = " OR ".join([f'"{item_id}"' for item_id in ids])
desired_state = {
'list': {
'settings_path': settings_path,
'custom_query': f'{to_property}({ids_clauses})',
'use_custom_query': True,
'search_term': "",
'at_least_one_facet_is_selected': False,
}
}
print('desired_state: ', desired_state)
b64_desired_state = base64.b64encode(json.dumps(desired_state).encode())
print('b64_desired_state: ', b64_desired_state)
print('destination_entity_browser_state_template: ', destination_entity_browser_state_template)
destination_entity_browser_state_url = destination_entity_browser_state_template.replace(
'<BROWSER_NAME>',
browser_name
)
destination_entity_browser_state_url = destination_entity_browser_state_url.replace(
'<GENERATED_STATE>',
b64_desired_state.decode()
)
print('destination_entity_browser_state_url: ', destination_entity_browser_state_url)
hashable_part = f'#{destination_entity_browser_state_url.split("#")[1]}'
print('hashable_part: ', hashable_part)
return get_destination_url_hash(hashable_part)
def get_destination_url_hash(hashable_part):
"""
:param hashable_part: of the destination url generated
:return: a hash generated using the hasable url part of the destination browser
"""
shortening_response = url_shortener.shorten_url(hashable_part)
return shortening_response['hash']
......@@ -73,7 +73,15 @@ JOIN_PROPERTIES = {
}
}
}
}
# This is used by the frontend to know which settings use to load the page, should be discarded in the future
SETTINGS_PATHS_FOR_TO_ENTITIES = {
PossibleEntitiesTo.CHEMBL_ACTIVITIES: 'ES_INDEXES_NO_MAIN_SEARCH.ACTIVITY',
}
BROWSER_NAMES_FOR_TO_ENTITIES = {
PossibleEntitiesTo.CHEMBL_ACTIVITIES: 'activities',
}
......@@ -101,3 +109,19 @@ def get_to_property(parsed_from_entity, parsed_to_entity):
get(parsed_from_entity, {}). \
get('to', {}). \
get(parsed_to_entity, {}).get('to_property')
def get_settings_path_for_to_entity(parsed_to_entity):
"""
:param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:return: the settings path used to create the destination page
"""
return SETTINGS_PATHS_FOR_TO_ENTITIES.get(parsed_to_entity)
def get_browser_names_for_to_entity(parsed_to_entity):
"""
:param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:return: the browser name used in the url for the destination page
"""
return BROWSER_NAMES_FOR_TO_ENTITIES.get(parsed_to_entity)
......@@ -11,14 +11,16 @@ from app import app_logging
from app.usage_statistics import statistics_saver
from app.url_shortening.expired_urls_deletion import ExpiredURLsDeletionThread
class URLNotFoundError(Exception):
"""
Error raises when an url was not found
"""
# ----------------------------------------------------------------------------------------------------------------------
# URL Shortening
# ----------------------------------------------------------------------------------------------------------------------
def shorten_url(long_url):
"""
:param long_url:
......
......@@ -12,7 +12,10 @@ from specific_tests import fun_test_simple_query, fun_test_query_with_context, \
fun_test_go_slim_target_classification, fun_test_in_vivo_assay_classification, \
fun_test_organism_taxonomy_target_classification, fun_test_protein_target_classification, \
fun_test_covid_entities_records, fun_test_database_summary, fun_test_entities_records, \
fun_test_get_all_properties, fun_test_identify_separator, fun_test_entities_join_0
fun_test_get_all_properties, fun_test_identify_separator
from specific_tests.entities_join import fun_test_entities_join_all_tests
PARSER = argparse.ArgumentParser()
PARSER.add_argument('server_base_path', help='server base path to run the tests against',
......@@ -28,7 +31,7 @@ def run():
"""
print(f'Running functional tests on {ARGS.server_base_path}')
for test_module in [fun_test_entities_join_0]:
for test_module in [fun_test_entities_join_all_tests]:
test_module.run_test(ARGS.server_base_path, ARGS.delayed_jobs_server_base_path)
......
{
"query": {
"bool": {
"filter": [
{
"bool": {
"should": [
{
"terms": {
"drug_warning.warning_class": [
"Hepatotoxicity"
]
}
}
]
}
},
{
"bool": {
"should": [
{
"terms": {
"drug_warning.where.country": [
"United Kingdom"
]
}
}
]
}
}
],
"should": [],
"must_not": []
}
},
"track_total_hits": true,
"size": 20,
"from": 0,
"_source": [
"drug_warning.molecule_chembl_id",
"drug_warning.parent_molecule_chembl_id",
"drug_warning.warning_type",
"drug_warning.warning_class",
"drug_warning.warning_description",
"drug_warning.where",
"drug_warning.warning_refs"
],
"sort": []
}
\ No newline at end of file
"""
Module that tests the endpoints to do joins among entities selecting all ids
"""
from specific_tests import utils
from specific_tests.entities_join import utils as entities_join_utils
def run_test(server_base_url, delayed_jobs_server_base_path):
"""
Tests doing a join among different entities selecting all ids
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print('-------------------------------------------')
print('Testing joins among entities')
print('-------------------------------------------')
dataset_query = utils.load_json_data('functional_tests/specific_tests/data/entities_join_query_0.json')
selection_description = {"selectionMode": "allItemsExcept", "exceptions": []}
entities_join_utils.test_entities_join(dataset_query, selection_description, server_base_url)
# pylint: disable=import-error
"""
Module that tests the endpoints to do joins among entities selecting all ids except some
"""
from specific_tests import utils
from specific_tests.entities_join import utils as entities_join_utils
def run_test(server_base_url, delayed_jobs_server_base_path):
"""
Tests doing a join among different entities selecting all ids except some
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print('-------------------------------------------')
print('Testing joins among entities selecting all except some')
print('-------------------------------------------')
dataset_query = utils.load_json_data('functional_tests/specific_tests/data/entities_join_query_0.json')
selection_description = {"selectionMode": "allItemsExcept", "exceptions": ['CHEMBL2107495', 'CHEMBL340978']}
entities_join_utils.test_entities_join(dataset_query, selection_description, server_base_url)
"""
Module that runs all the tests related to the entities join
"""
from specific_tests.entities_join import fun_test_entities_join_0, fun_test_entities_join_1
def run_test(server_base_url, delayed_jobs_server_base_path):
"""
Tests doing a join among different entities selecting all ids
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
for test_module in [fun_test_entities_join_0, fun_test_entities_join_1]:
test_module.run_test(server_base_url, delayed_jobs_server_base_path)
"""
Module with utils for entities join tests
"""
import json
import requests
from specific_tests import utils
def test_entities_join(dataset_query, selection_description, server_base_url):
"""
Performs the testing of the entities join endpoint given the parameters
:param dataset_query: query of the current dataset
:param selection_description: dict describing the selection
:param server_base_url: base url of the server being used for the tests
"""
join_params = {
'destination_entity_browser_state_template':
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>',
'entity_from': 'CHEMBL_DRUG_WARNINGS',
'entity_to': 'CHEMBL_ACTIVITIES',
'es_query': json.dumps(dataset_query),
'selection_description': json.dumps(selection_description)
}
url = f'{server_base_url}/entities_join/get_link_to_related_items'
request = requests.post(url, data=join_params)
status_code = request.status_code
print(f'status_code: {status_code}')
response_text = request.text
utils.print_es_response(response_text)
assert status_code == 200, 'The request failed!'
url_hash_got = request.json()['tiny_hash']
print('url_hash_got: ', url_hash_got)
expansion_url = f'{server_base_url}/url_shortening/expand_url/{url_hash_got}'
print(f'Checking tiny hash: {expansion_url}')
request = requests.get(expansion_url)
status_code = request.status_code
print(f'status_code: {status_code}')
response_text = request.text
utils.print_es_response(response_text)
assert status_code == 200, 'The request failed!'
# pylint: disable=import-error
"""
Module that tests the endpoints to do joins among entities selecting all ids
"""
import json
import requests
from specific_tests import utils
def run_test(server_base_url, delayed_jobs_server_base_path):
"""
Tests doing a join among different entities selecting all ids
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print('-------------------------------------------')
print('Testing joins among entities')
print('-------------------------------------------')
dataset_query = {
"query": {
"bool": {
"filter": [
{
"bool": {
"should": [
{
"terms": {
"drug_warning.warning_class": [
"Hepatotoxicity"
]
}
}
]
}
},
{
"bool": {
"should": [
{
"terms": {
"drug_warning.where.country": [
"United Kingdom"
]
}
}
]
}
}
],
"should": [],
"must_not": []
}
},
"track_total_hits": True,
"size": 20,
"from": 0,
"_source": [
"drug_warning.molecule_chembl_id",
"drug_warning.parent_molecule_chembl_id",
"drug_warning.warning_type",
"drug_warning.warning_class",
"drug_warning.warning_description",
"drug_warning.where",
"drug_warning.warning_refs"
],
"sort": []
}
selection_description = {"selectionMode": "allItemsExcept", "exceptions": []}
join_params = {
'destination_entity_browser_state_template': 'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>',
'entity_from': 'CHEMBL_DRUG_WARNINGS',
'entity_to': 'CHEMBL_ACTIVITIES',
'es_query': json.dumps(dataset_query),
'selection_description': json.dumps(selection_description)
}
url = f'{server_base_url}/entities_join/get_link_to_related_items'
print('doing post: ', url)
request = requests.post(url, data=join_params)
print('post done!')
status_code = request.status_code
print(f'status_code: {status_code}')
response_text = request.text
utils.print_es_response(response_text)
assert status_code == 200, 'The request failed!'
......@@ -50,7 +50,7 @@ def run_test(server_base_url, delayed_jobs_server_base_path):
utils.print_es_response(response_text)
assert status_code == 200, 'The request failed!'
time.sleep(5) # make it sleep to make sure the url is saved in ES
time.sleep(5) # make it sleep to make sure the url is saved in ES
response_json = request.json()
long_url_got = response_json.get('long_url')
......
"""
Module with utils functions for the tests
"""
import json
import time
import requests
......@@ -167,3 +168,12 @@ def wait_until_job_finished(delayed_jobs_server_base_path, job_id):
print('job_status: ', job_status)
print('job_progress: ', job_progress)
time.sleep(1)
def load_json_data(file_path):
"""
:param file_path: path of the json file to load
:return: a dict with the data loaded
"""
with open(file_path) as the_file:
return json.loads(the_file.read())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment