Entities join endpoint: add more functional tests

.gitignore

@@ -139,4 +139,5 @@ app/static/
 
 output.csv
 
+deleteme
 deleteme.json
\ No newline at end of file

app/entities_joiner/entities_joiner.py

@@ -2,9 +2,11 @@
 Module with functions to make joins of entities
 """
 import json
+import base64
 
 from app.entities_joiner import standardisation
 from app.entities_joiner import ids_loader
+from app.url_shortening import url_shortener
 
 
 class EntitiesJoinerError(Exception):
@@ -53,15 +55,11 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
     print('load ids:')
 
     index_name = standardisation.get_index_name_for_from_entity(parsed_from_entity)
-
-    if index_name is None:
-        raise EntitiesJoinerError(f'There is no index name configured for queries from {parsed_from_entity.value}')
+    fail_if_null(index_name, 'index name', parsed_from_entity, parsed_to_entity)
 
     print('index_name: ', index_name)
     from_property = standardisation.get_from_property(parsed_from_entity, parsed_to_entity)
-
-    if from_property is None:
-        raise EntitiesJoinerError(f'There is no from property configured for queries from {parsed_from_entity.value}')
+    fail_if_null(from_property, 'from property', parsed_from_entity, parsed_to_entity)
 
     print('from_property: ', from_property)
     selection_description = json.loads(raw_selection_description)
@@ -69,18 +67,43 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
     ids = ids_loader.load_ids_for_query(es_query, selection_description, from_property, index_name)
 
     to_property = standardisation.get_to_property(parsed_from_entity, parsed_to_entity)
-    join_query = get_join_query(ids, to_property)
-    print('join_query: ')
-    print(join_query)
+    fail_if_null(to_property, 'to property', parsed_from_entity, parsed_to_entity)
+
+    settings_path = standardisation.get_settings_path_for_to_entity(parsed_to_entity)
+    fail_if_null(settings_path, 'settings path', parsed_from_entity, parsed_to_entity)
+
+    browser_name = standardisation.get_browser_names_for_to_entity(parsed_to_entity)
+    fail_if_null(browser_name, 'browser name', parsed_from_entity, parsed_to_entity)
+
+    join_state_hash = get_join_state_hash(ids, to_property, settings_path, browser_name,
+                                          destination_entity_browser_state_template)
+    print('join_state_hash: ')
+    print(join_state_hash)
+
+    return join_state_hash
 
-    return 'holaaaa'
+
+def fail_if_null(value, value_name, parsed_from_entity, parsed_to_entity):
+    """
+    Fails if the value is null and uses the rest of the parameters to build an error message
+    :param value: value to check
+    :param value_name: name of the value for the error message
+    :param parsed_from_entity: parsed from entity
+    :param parsed_to_entity: parsed to entity
+    """
+    if value is None:
+        raise EntitiesJoinerError(f'There is no {value_name} configured for queries from {parsed_from_entity.value} '
+                                  f'to {parsed_to_entity.value}')
 
 
-def get_join_query(ids, to_property):
+def get_join_state_hash(ids, to_property, settings_path, browser_name, destination_entity_browser_state_template):
     """
     :param ids: list of its for the join
     :param to_property: entity to which to do the join
-    :return: the query to use for the join
+    :param settings_path: settings path to be used by the user interface
+    :param browser_name: name of the destination browser used to build the desired url
+    :param destination_entity_browser_state_template: template to build the url for the destination entity browser
+    :return: the hash for the query state used for the join
     """
     print('get_join_query')
     print('ids: ')
@@ -88,11 +111,41 @@ def get_join_query(ids, to_property):
     print('to_property: ')
     print(to_property)
 
-    return
-    {
-        "query": {
-            "terms": {
-                "molecule_chembl_id": ["CHEMBL2107495", "CHEMBL1204165"]
-            }
+    ids_clauses = " OR ".join([f'"{item_id}"' for item_id in ids])
+    desired_state = {
+        'list': {
+            'settings_path': settings_path,
+            'custom_query': f'{to_property}({ids_clauses})',
+            'use_custom_query': True,
+            'search_term': "",
+            'at_least_one_facet_is_selected': False,
         }
     }
+    print('desired_state: ', desired_state)
+    b64_desired_state = base64.b64encode(json.dumps(desired_state).encode())
+    print('b64_desired_state: ', b64_desired_state)
+    print('destination_entity_browser_state_template: ', destination_entity_browser_state_template)
+    destination_entity_browser_state_url = destination_entity_browser_state_template.replace(
+        '<BROWSER_NAME>',
+        browser_name
+    )
+    destination_entity_browser_state_url = destination_entity_browser_state_url.replace(
+        '<GENERATED_STATE>',
+        b64_desired_state.decode()
+    )
+
+    print('destination_entity_browser_state_url: ', destination_entity_browser_state_url)
+    hashable_part = f'#{destination_entity_browser_state_url.split("#")[1]}'
+    print('hashable_part: ', hashable_part)
+
+    return get_destination_url_hash(hashable_part)
+
+
+def get_destination_url_hash(hashable_part):
+    """
+    :param hashable_part: of the destination url generated
+    :return: a hash generated using the hasable url part of the destination browser
+    """
+    shortening_response = url_shortener.shorten_url(hashable_part)
+    return shortening_response['hash']
+

app/entities_joiner/standardisation.py

@@ -73,7 +73,15 @@ JOIN_PROPERTIES = {
             }
         }
     }
+}
+
+# This is used by the frontend to know which settings use to load the page, should be discarded in the future
+SETTINGS_PATHS_FOR_TO_ENTITIES = {
+    PossibleEntitiesTo.CHEMBL_ACTIVITIES: 'ES_INDEXES_NO_MAIN_SEARCH.ACTIVITY',
+}
 
+BROWSER_NAMES_FOR_TO_ENTITIES = {
+    PossibleEntitiesTo.CHEMBL_ACTIVITIES: 'activities',
 }
 
 
@@ -101,3 +109,19 @@ def get_to_property(parsed_from_entity, parsed_to_entity):
         get(parsed_from_entity, {}). \
         get('to', {}). \
         get(parsed_to_entity, {}).get('to_property')
+
+
+def get_settings_path_for_to_entity(parsed_to_entity):
+    """
+    :param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
+    :return: the settings path used to create the destination page
+    """
+    return SETTINGS_PATHS_FOR_TO_ENTITIES.get(parsed_to_entity)
+
+
+def get_browser_names_for_to_entity(parsed_to_entity):
+    """
+    :param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
+    :return: the browser name used in the url for the destination page
+    """
+    return BROWSER_NAMES_FOR_TO_ENTITIES.get(parsed_to_entity)

app/url_shortening/url_shortener.py

@@ -11,14 +11,16 @@ from app import app_logging
 from app.usage_statistics import statistics_saver
 from app.url_shortening.expired_urls_deletion import ExpiredURLsDeletionThread
 
+
 class URLNotFoundError(Exception):
     """
     Error raises when an url was not found
     """
+
+
 # ----------------------------------------------------------------------------------------------------------------------
 # URL Shortening
 # ----------------------------------------------------------------------------------------------------------------------
-
 def shorten_url(long_url):
     """
     :param long_url:

functional_tests/run_functional_tests.py

@@ -12,7 +12,10 @@ from specific_tests import fun_test_simple_query, fun_test_query_with_context, \
     fun_test_go_slim_target_classification, fun_test_in_vivo_assay_classification, \
     fun_test_organism_taxonomy_target_classification, fun_test_protein_target_classification, \
     fun_test_covid_entities_records, fun_test_database_summary, fun_test_entities_records, \
-    fun_test_get_all_properties, fun_test_identify_separator, fun_test_entities_join_0
+    fun_test_get_all_properties, fun_test_identify_separator
+
+
+from specific_tests.entities_join import fun_test_entities_join_all_tests
 
 PARSER = argparse.ArgumentParser()
 PARSER.add_argument('server_base_path', help='server base path to run the tests against',
@@ -28,7 +31,7 @@ def run():
     """
     print(f'Running functional tests on {ARGS.server_base_path}')
 
-    for test_module in [fun_test_entities_join_0]:
+    for test_module in [fun_test_entities_join_all_tests]:
         test_module.run_test(ARGS.server_base_path, ARGS.delayed_jobs_server_base_path)
 
 

functional_tests/specific_tests/data/entities_join_query_0.json

+{
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "bool": {
+            "should": [
+              {
+                "terms": {
+                  "drug_warning.warning_class": [
+                    "Hepatotoxicity"
+                  ]
+                }
+              }
+            ]
+          }
+        },
+        {
+          "bool": {
+            "should": [
+              {
+                "terms": {
+                  "drug_warning.where.country": [
+                    "United Kingdom"
+                  ]
+                }
+              }
+            ]
+          }
+        }
+      ],
+      "should": [],
+      "must_not": []
+    }
+  },
+  "track_total_hits": true,
+  "size": 20,
+  "from": 0,
+  "_source": [
+    "drug_warning.molecule_chembl_id",
+    "drug_warning.parent_molecule_chembl_id",
+    "drug_warning.warning_type",
+    "drug_warning.warning_class",
+    "drug_warning.warning_description",
+    "drug_warning.where",
+    "drug_warning.warning_refs"
+  ],
+  "sort": []
+}
\ No newline at end of file

functional_tests/specific_tests/entities_join/fun_test_entities_join_0.py

+"""
+Module that tests the endpoints to do joins among entities selecting all ids
+"""
+from specific_tests import utils
+from specific_tests.entities_join import utils as entities_join_utils
+
+
+def run_test(server_base_url, delayed_jobs_server_base_path):
+    """
+    Tests doing a join among different entities selecting all ids
+    :param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
+    :param delayed_jobs_server_base_path: base path for the delayed_jobs
+    """
+    print('-------------------------------------------')
+    print('Testing joins among entities')
+    print('-------------------------------------------')
+
+    dataset_query = utils.load_json_data('functional_tests/specific_tests/data/entities_join_query_0.json')
+    selection_description = {"selectionMode": "allItemsExcept", "exceptions": []}
+    entities_join_utils.test_entities_join(dataset_query, selection_description, server_base_url)

functional_tests/specific_tests/entities_join/fun_test_entities_join_1.py

+# pylint: disable=import-error
+"""
+Module that tests the endpoints to do joins among entities selecting all ids except some
+"""
+from specific_tests import utils
+from specific_tests.entities_join import utils as entities_join_utils
+
+
+def run_test(server_base_url, delayed_jobs_server_base_path):
+    """
+    Tests doing a join among different entities selecting all ids except some
+    :param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
+    :param delayed_jobs_server_base_path: base path for the delayed_jobs
+    """
+
+    print('-------------------------------------------')
+    print('Testing joins among entities selecting all except some')
+    print('-------------------------------------------')
+    dataset_query = utils.load_json_data('functional_tests/specific_tests/data/entities_join_query_0.json')
+    selection_description = {"selectionMode": "allItemsExcept", "exceptions": ['CHEMBL2107495', 'CHEMBL340978']}
+    entities_join_utils.test_entities_join(dataset_query, selection_description, server_base_url)

functional_tests/specific_tests/entities_join/fun_test_entities_join_all_tests.py

+"""
+Module that runs all the tests related to the entities join
+"""
+from specific_tests.entities_join import fun_test_entities_join_0, fun_test_entities_join_1
+
+
+def run_test(server_base_url, delayed_jobs_server_base_path):
+    """
+    Tests doing a join among different entities selecting all ids
+    :param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
+    :param delayed_jobs_server_base_path: base path for the delayed_jobs
+    """
+    for test_module in [fun_test_entities_join_0, fun_test_entities_join_1]:
+        test_module.run_test(server_base_url, delayed_jobs_server_base_path)

functional_tests/specific_tests/entities_join/utils.py

+"""
+Module with utils for entities join tests
+"""
+import json
+import requests
+
+from specific_tests import utils
+
+
+def test_entities_join(dataset_query, selection_description, server_base_url):
+    """
+    Performs the testing of the entities join endpoint given the parameters
+    :param dataset_query: query of the current dataset
+    :param selection_description: dict describing the selection
+    :param server_base_url: base url of the server being used for the tests
+    """
+
+    join_params = {
+        'destination_entity_browser_state_template':
+            'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>',
+        'entity_from': 'CHEMBL_DRUG_WARNINGS',
+        'entity_to': 'CHEMBL_ACTIVITIES',
+        'es_query': json.dumps(dataset_query),
+        'selection_description': json.dumps(selection_description)
+    }
+
+    url = f'{server_base_url}/entities_join/get_link_to_related_items'
+    request = requests.post(url, data=join_params)
+
+    status_code = request.status_code
+    print(f'status_code: {status_code}')
+    response_text = request.text
+    utils.print_es_response(response_text)
+    assert status_code == 200, 'The request failed!'
+
+    url_hash_got = request.json()['tiny_hash']
+    print('url_hash_got: ', url_hash_got)
+
+    expansion_url = f'{server_base_url}/url_shortening/expand_url/{url_hash_got}'
+    print(f'Checking tiny hash: {expansion_url}')
+    request = requests.get(expansion_url)
+
+    status_code = request.status_code
+    print(f'status_code: {status_code}')
+    response_text = request.text
+    utils.print_es_response(response_text)
+    assert status_code == 200, 'The request failed!'

functional_tests/specific_tests/fun_test_entities_join_0.py

-# pylint: disable=import-error
-"""
-Module that tests the endpoints to do joins among entities selecting all ids
-"""
-import json
-import requests
-
-from specific_tests import utils
-
-
-def run_test(server_base_url, delayed_jobs_server_base_path):
-    """
-    Tests doing a join among different entities selecting all ids
-    :param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
-    :param delayed_jobs_server_base_path: base path for the delayed_jobs
-    """
-
-    print('-------------------------------------------')
-    print('Testing joins among entities')
-    print('-------------------------------------------')
-    dataset_query = {
-        "query": {
-            "bool": {
-                "filter": [
-                    {
-                        "bool": {
-                            "should": [
-                                {
-                                    "terms": {
-                                        "drug_warning.warning_class": [
-                                            "Hepatotoxicity"
-                                        ]
-                                    }
-                                }
-                            ]
-                        }
-                    },
-                    {
-                        "bool": {
-                            "should": [
-                                {
-                                    "terms": {
-                                        "drug_warning.where.country": [
-                                            "United Kingdom"
-                                        ]
-                                    }
-                                }
-                            ]
-                        }
-                    }
-                ],
-                "should": [],
-                "must_not": []
-            }
-        },
-        "track_total_hits": True,
-        "size": 20,
-        "from": 0,
-        "_source": [
-            "drug_warning.molecule_chembl_id",
-            "drug_warning.parent_molecule_chembl_id",
-            "drug_warning.warning_type",
-            "drug_warning.warning_class",
-            "drug_warning.warning_description",
-            "drug_warning.where",
-            "drug_warning.warning_refs"
-        ],
-        "sort": []
-    }
-
-    selection_description = {"selectionMode": "allItemsExcept", "exceptions": []}
-
-    join_params = {
-        'destination_entity_browser_state_template': 'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>',
-        'entity_from': 'CHEMBL_DRUG_WARNINGS',
-        'entity_to': 'CHEMBL_ACTIVITIES',
-        'es_query': json.dumps(dataset_query),
-        'selection_description': json.dumps(selection_description)
-    }
-
-    url = f'{server_base_url}/entities_join/get_link_to_related_items'
-    print('doing post: ', url)
-    request = requests.post(url, data=join_params)
-    print('post done!')
-
-    status_code = request.status_code
-    print(f'status_code: {status_code}')
-    response_text = request.text
-    utils.print_es_response(response_text)
-    assert status_code == 200, 'The request failed!'

functional_tests/specific_tests/fun_test_url_shortening.py

@@ -50,7 +50,7 @@ def run_test(server_base_url, delayed_jobs_server_base_path):
     utils.print_es_response(response_text)
     assert status_code == 200, 'The request failed!'
 
-    time.sleep(5) # make it sleep to make sure the url is saved in ES
+    time.sleep(5)  # make it sleep to make sure the url is saved in ES
 
     response_json = request.json()
     long_url_got = response_json.get('long_url')

functional_tests/specific_tests/utils.py

 """
 Module with utils functions for the tests
 """
+import json
 import time
 
 import requests
@@ -167,3 +168,12 @@ def wait_until_job_finished(delayed_jobs_server_base_path, job_id):
         print('job_status: ', job_status)
         print('job_progress: ', job_progress)
         time.sleep(1)
+
+
+def load_json_data(file_path):
+    """
+    :param file_path: path of the json file to load
+    :return: a dict with the data loaded
+    """
+    with open(file_path) as the_file:
+        return json.loads(the_file.read())