Entities Join: Implement generation of query to get all ids

app/blueprints/entities_join/controllers/entities_join_controller.py

@@ -40,4 +40,7 @@ def get_link_to_related_items():
         return jsonify(json_response)
     except entities_join_service.EntitiesJoinServiceError as error:
 
-        abort(500, msg=f'Internal server error: {str(error)}')
+        print('error!')
+        print(error)
+
+        abort(500, f'Internal server error: {str(error)}')

app/blueprints/entities_join/services/entities_join_service.py

 """
 Service that handles the requests to the entities join
 """
-import json
-
-from app import app_logging
-from app.es_data import es_data
-from app.context_loader import context_loader
-from app.config import RUN_CONFIG
-from app.es_data import es_mappings
+from app.entities_joiner import entities_joiner
 
 
 class EntitiesJoinServiceError(Exception):
@@ -23,7 +17,17 @@ def get_link_to_related_items(destination_entity_browser_state_template,
     :param entity_to: destination entity of the join
     :param es_query: query in elasticsearch for the dataset
     :param selection_description: stringifyed javascript object describing de selection of items in the dataset
+    :return: a dict with the tiny url to the link with the generated state
     """
-    return {
-        'msg': 'hola'
-    }
+
+    try:
+        tiny_hash = entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
+                                                                   entity_from, entity_to, es_query,
+                                                                   selection_description)
+
+        return {
+            'tiny_hash': tiny_hash
+        }
+
+    except entities_joiner.EntitiesJoinerError as error:
+        raise EntitiesJoinServiceError(error)

app/entities_joiner/entities_joiner.py

+"""
+Module with functions to make joins of entities
+"""
+import json
+
+from app.entities_joiner import standardisation
+
+
+class EntitiesJoinerError(Exception):
+    """Base class for exceptions in this file."""
+
+
+def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
+                                   entity_from, entity_to, es_query,
+                                   selection_description):
+    """
+    :param destination_entity_browser_state_template: template for building the resulting browser url
+    :param entity_from: source entity of the items
+    :param entity_to: destination entity of the join
+    :param es_query: query in elasticsearch for the dataset
+    :param selection_description: stringifyed javascript object describing de selection of items in the dataset
+    :return: the hash to the link with the generated state
+    """
+    parsed_from = None
+    try:
+        parsed_from = standardisation.PossibleFroms(entity_from)
+    except ValueError as error:
+        raise EntitiesJoinerError(
+            f'entity_from: {str(error)}. Possible values are {[item.value for item in standardisation.PossibleFroms]}')
+
+    parsed_to = None
+    try:
+        parsed_to = standardisation.PossibleTos(entity_to)
+    except ValueError as error:
+        raise EntitiesJoinerError(
+            f'entity_to: {str(error)}. Possible values are {[item.value for item in standardisation.PossibleTos]}')
+
+    selection_description_dict = json.loads(selection_description)
+    try:
+        parsed_selection_mode = standardisation.SelectionModes(selection_description_dict['selectionMode'])
+        if parsed_selection_mode == standardisation.SelectionModes.NO_ITEMS_EXCEPT:
+            if len(selection_description_dict.get('exceptions', [])) == 0:
+                raise EntitiesJoinerError(
+                    f'When selection mode is {parsed_selection_mode} there must be at least one exception')
+    except ValueError as error:
+        raise EntitiesJoinerError(
+            f'selectionMode: {str(error)}. Possible values are {[item.value for item in standardisation.SelectionModes]}')
+
+    if entity_to == entity_from:
+        raise EntitiesJoinerError(f'entity_to ({entity_to}) and entity_from ({entity_from}) cannot be the same!')
+
+    return 'holaaaa'

app/entities_joiner/ids_loader.py

+"""
+Module with functions that help to load the ids for the entities joiner
+"""
+from app.entities_joiner import standardisation
+
+
+def get_ids_query(es_query, selection_description, from_property):
+    """
+    :param es_query: query for the dataset
+    :param selection_description: dict describing the selection
+    :param from_property: property to get to to the join
+    :return: the query to use to get the ids depending on the selection description
+    """
+    selection_mode = selection_description['selectionMode']
+    parsed_selection_mode = standardisation.SelectionModes(selection_mode)
+    exceptions = selection_description.get('exceptions', [])
+
+    if parsed_selection_mode == standardisation.SelectionModes.ALL_ITEMS_EXCEPT:
+        if len(exceptions) == 0:
+            return {
+                'query': es_query.get('query'),
+                "_source": [from_property],
+            }
+        else:
+            dataset_query = es_query.get('query')
+            if dataset_query.get('bool') is None:
+                dataset_query['bool'] = {}
+            if dataset_query.get('bool').get('must_not') is None:
+                dataset_query['bool']['must_not'] = []
+            dataset_query['bool']['must_not'].append({
+                'terms': {
+                    from_property: exceptions
+                }
+            })
+            return {
+                'query': dataset_query,
+                "_source": [from_property],
+            }

app/entities_joiner/standardisation.py

+"""
+Module that helps with the standardisation of the package
+"""
+from enum import Enum
+
+
+class PossibleFroms(Enum):
+    """
+    Enumeration with the possible froms allowed
+    """
+    CHEMBL_ACTIVITIES = 'CHEMBL_ACTIVITIES'
+    CHEMBL_COMPOUNDS = 'CHEMBL_COMPOUNDS'
+    CHEMBL_TARGETS = 'CHEMBL_TARGETS'
+    CHEMBL_ASSAYS = 'CHEMBL_ASSAYS'
+    CHEMBL_DOCUMENTS = 'CHEMBL_DOCUMENTS'
+    CHEMBL_CELL_LINES = 'CHEMBL_CELL_LINES'
+    CHEMBL_TISSUES = 'CHEMBL_TISSUES'
+    CHEMBL_DRUG_WARNINGS = 'CHEMBL_DRUG_WARNINGS'
+
+
+class PossibleTos(Enum):
+    """
+    Enumeration with the possible to allowed
+    """
+    CHEMBL_ACTIVITIES = 'CHEMBL_ACTIVITIES'
+    CHEMBL_COMPOUNDS = 'CHEMBL_COMPOUNDS'
+    CHEMBL_TARGETS = 'CHEMBL_TARGETS'
+    CHEMBL_ASSAYS = 'CHEMBL_ASSAYS'
+    CHEMBL_DOCUMENTS = 'CHEMBL_DOCUMENTS'
+    CHEMBL_CELL_LINES = 'CHEMBL_CELL_LINES'
+    CHEMBL_TISSUES = 'CHEMBL_TISSUES'
+    CHEMBL_DRUG_WARNINGS = 'CHEMBL_DRUG_WARNINGS'
+
+
+class SelectionModes(Enum):
+    """
+    Possible selection modes
+    """
+    ALL_ITEMS_EXCEPT = 'allItemsExcept'
+    NO_ITEMS_EXCEPT = 'noItemsExcept'

app/entities_joiner/test/test_entities_joiner.py

+"""
+Module to test the entities joiner module
+"""
+import unittest
+
+from app.entities_joiner import entities_joiner
+
+
+class TestEntitiesJoiner(unittest.TestCase):
+    """
+    Class to test the entities joiner module
+    """
+
+    def test_fails_when_from_and_to_are_equal(self):
+        """
+        Tests that it fails when from and to are equal
+        """
+        destination_entity_browser_state_template = 'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state' \
+                                                    '/<GENERATED_STATE> '
+        entity_from = 'CHEMBL_COMPOUNDS'
+        entity_to = 'CHEMBL_COMPOUNDS'
+        es_query = '{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, ' \
+                   '"query": "*"}}], "filter": []}}, "sort": []} '
+        selection_description = '{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", ' \
+                                '"CHEMBL64"]} '
+
+        with self.assertRaises(entities_joiner.EntitiesJoinerError):
+            entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
+                                                           entity_from, entity_to, es_query,
+                                                           selection_description)
+
+    def test_fails_when_from_is_not_on_the_possibilities(self):
+        """
+        Tests that it fails when from is not on the possibilities
+        """
+
+        destination_entity_browser_state_template = 'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state' \
+                                                    '/<GENERATED_STATE> '
+        entity_from = 'NON_EXISTENT'
+        entity_to = 'CHEMBL_COMPOUNDS'
+        es_query = '{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, ' \
+                   '"query": "*"}}], "filter": []}}, "sort": []} '
+        selection_description = '{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", ' \
+                                '"CHEMBL64"]} '
+
+        with self.assertRaises(entities_joiner.EntitiesJoinerError):
+            entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
+                                                           entity_from, entity_to, es_query,
+                                                           selection_description)
+
+    def test_fails_when_to_is_not_on_the_possibilities(self):
+        """
+        Tests that it fails when to is not on the possibilities
+        """
+
+        destination_entity_browser_state_template = 'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state' \
+                                                    '/<GENERATED_STATE> '
+        entity_from = 'CHEMBL_COMPOUNDS'
+        entity_to = 'NON_EXISTENT'
+        es_query = '{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, ' \
+                   '"query": "*"}}], "filter": []}}, "sort": []} '
+        selection_description = '{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", ' \
+                                '"CHEMBL64"]} '
+
+        with self.assertRaises(entities_joiner.EntitiesJoinerError):
+            entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
+                                                           entity_from, entity_to, es_query,
+                                                           selection_description)
+
+    def test_fails_when_selection_mode_not_in_possibilities(self):
+        """
+        Tests that it fails when to is not on the possibilities
+        """
+
+        destination_entity_browser_state_template = 'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state' \
+                                                    '/<GENERATED_STATE> '
+        entity_from = 'CHEMBL_COMPOUNDS'
+        entity_to = 'CHEMBL_TARGETS'
+        es_query = '{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, ' \
+                   '"query": "*"}}], "filter": []}}, "sort": []} '
+        selection_description = '{"selectionMode": "non_existent","exceptions": ["CHEMBL3989861", "CHEMBL3989724", ' \
+                                '"CHEMBL64"]} '
+
+        with self.assertRaises(entities_joiner.EntitiesJoinerError):
+            entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
+                                                           entity_from, entity_to, es_query,
+                                                           selection_description)
+
+    def test_fails_when_selecting_no_items(self):
+        """
+        Tests that it fails when no items are selected
+        """
+        destination_entity_browser_state_template = 'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state' \
+                                                    '/<GENERATED_STATE> '
+        entity_from = 'CHEMBL_COMPOUNDS'
+        entity_to = 'CHEMBL_TARGETS'
+        es_query = '{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, ' \
+                   '"query": "*"}}], "filter": []}}, "sort": []} '
+        selection_description = '{"selectionMode": "noItemsExcept","exceptions": []} '
+
+        with self.assertRaises(entities_joiner.EntitiesJoinerError):
+            entities_joiner.get_tiny_hash_to_related_items(destination_entity_browser_state_template,
+                                                           entity_from, entity_to, es_query,
+                                                           selection_description)

app/entities_joiner/test/test_ids_loader.py

+"""
+Module to test the ids loader
+"""
+import unittest
+
+from app.entities_joiner import ids_loader
+
+
+class TestIDsLoader(unittest.TestCase):
+    """
+    Class to test the ids loader module
+    """
+
+    def test_generates_query_to_get_ids_when_selecting_all(self):
+        """
+        test that it generates the required query when selected all items in dataset
+        """
+        es_query = {
+            "query": {
+                "bool": {
+                    "filter": [],
+                    "should": [],
+                    "must_not": []
+                }
+            },
+            "track_total_hits": True,
+            "size": 20,
+            "from": 0,
+            "_source": [
+                "drug_warning.molecule_chembl_id",
+                "drug_warning.parent_molecule_chembl_id",
+                "drug_warning.warning_type",
+                "drug_warning.warning_class",
+                "drug_warning.warning_description",
+                "drug_warning.where",
+                "drug_warning.warning_refs"
+            ],
+            "sort": []
+        }
+
+        selection_description = {
+            "selectionMode": "allItemsExcept",
+            "exceptions": []
+        }
+
+        from_property = 'drug_warning.molecule_chembl_id'
+
+        ids_query_must_be = {
+            "query": {
+                "bool": {
+                    "filter": [],
+                    "should": [],
+                    "must_not": []
+                }
+            },
+            "_source": [from_property],
+        }
+
+        ids_query_got = ids_loader.get_ids_query(es_query, selection_description, from_property)
+        self.assertDictEqual(ids_query_must_be, ids_query_got,
+                             msg='The query was not generated correctly when selecting all items!')
+
+    def test_generates_query_to_get_ids_when_selecting_all_except_some(self):
+        """
+        test that it generates the required query when selected all items in dataset
+        """
+        es_query = {
+            "query": {
+                "bool": {
+                    "filter": [],
+                    "should": [],
+                    "must_not": []
+                }
+            },
+            "track_total_hits": True,
+            "size": 20,
+            "from": 0,
+            "_source": [
+                "drug_warning.molecule_chembl_id",
+                "drug_warning.parent_molecule_chembl_id",
+                "drug_warning.warning_type",
+                "drug_warning.warning_class",
+                "drug_warning.warning_description",
+                "drug_warning.where",
+                "drug_warning.warning_refs"
+            ],
+            "sort": []
+        }
+
+        selection_description = {
+            "selectionMode": "allItemsExcept",
+            "exceptions": ['CHEMBL3989861', 'CHEMBL3989724', 'CHEMBL64']
+        }
+
+        from_property = 'drug_warning.molecule_chembl_id'
+
+        ids_query_must_be = {
+            "query": {
+                "bool": {
+                    "filter": [],
+                    "should": [],
+                    "must_not": [
+                        {
+                            "terms": {
+                                from_property: selection_description['exceptions']
+                            }
+                        }
+                    ]
+                }
+            },
+            "_source": [from_property],
+        }
+
+        ids_query_got = ids_loader.get_ids_query(es_query, selection_description, from_property)
+        self.assertDictEqual(ids_query_must_be, ids_query_got,
+                             msg='The query was not generated correctly when selecting all items except some!')
+
+    def test_generates_query_to_get_ids_when_selecting_none(self):
+        """
+        test that it generates the required query when select in dataset
+        """

app/swagger/swagger.yaml

@@ -515,14 +515,14 @@ paths:
           description: 'Source entity for the join.'
           required: true
           type: 'string'
-          enum: [ 'CHEMBL_COMPOUNDS', 'CHEMBL_TARGETS', 'CHEMBL_ASSAYS', 'CHEMBL_DOCUMENTS', 'CHEMBL_CELL_LINES', 'CHEMBL_TISSUES' ]
+          enum: [ 'CHEMBL_ACTIVITIES', 'CHEMBL_COMPOUNDS', 'CHEMBL_TARGETS', 'CHEMBL_ASSAYS', 'CHEMBL_DOCUMENTS', 'CHEMBL_CELL_LINES', 'CHEMBL_TISSUES', 'CHEMBL_DRUG_WARNINGS' ]
           default: 'CHEMBL_COMPOUNDS'
         - name: 'entity_to'
           in: 'formData'
           description: 'The entity in ChEMBL to do the join with.'
           required: true
           type: 'string'
-          enum: [ 'CHEMBL_COMPOUNDS', 'CHEMBL_TARGETS', 'CHEMBL_ASSAYS', 'CHEMBL_DOCUMENTS', 'CHEMBL_CELL_LINES', 'CHEMBL_TISSUES' ]
+          enum: [ 'CHEMBL_ACTIVITIES', 'CHEMBL_COMPOUNDS', 'CHEMBL_TARGETS', 'CHEMBL_ASSAYS', 'CHEMBL_DOCUMENTS', 'CHEMBL_CELL_LINES', 'CHEMBL_TISSUES', 'CHEMBL_DRUG_WARNINGS' ]
           default: 'CHEMBL_COMPOUNDS'
         - name: 'es_query'
           in: 'formData'

functional_tests/specific_tests/fun_test_entities_join_0.py

+# pylint: disable=import-error
+"""
+Module that tests the endpoints to do joins among entities
+"""
+import requests
+
+
+def run_test(server_base_url, delayed_jobs_server_base_path):
+    """
+    Tests doing a join among different entities
+    :param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
+    :param delayed_jobs_server_base_path: base path for the delayed_jobs
+    """
+
+    print('-------------------------------------------')
+    print('Testing joins among entities')
+    print('-------------------------------------------')
+
+    join_params = {
+        'destination_entity_browser_state_template': 'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>',
+        'entity_from': 'CHEMBL_DRUG_WARNINGS',
+        'entity_to': 'CHEMBL_ACTIVITIES',
+        'es_query': '',
+    }
+
+    url = f'{server_base_url}/entities_join/get_link_to_related_items'

functional_tests/specific_tests/fun_test_get_context_data.py

@@ -9,7 +9,7 @@ from specific_tests import utils
 
 def run_test(server_base_url, delayed_jobs_server_base_path):
     """
-    Tests that getting the data of a context
+    Tests that getting the data of a context works
     :param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
     :param delayed_jobs_server_base_path: base path for the delayed_jobs
     """