Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
ChEMBL
C
ChEMBL
Main Web Interface
Elasticsearch Proxy API
Commits
23b2075a
Commit
23b2075a
authored
Jun 10, 2021
by
David Mendez
Browse files
Entities Join: Implement generation of query to get all ids
parent
05590d3e
Changes
12
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
402 additions
and
14 deletions
+402
-14
app/blueprints/entities_join/controllers/entities_join_controller.py
...nts/entities_join/controllers/entities_join_controller.py
+4
-1
app/blueprints/entities_join/services/entities_join_service.py
...lueprints/entities_join/services/entities_join_service.py
+14
-10
app/entities_joiner/__init__.py
app/entities_joiner/__init__.py
+0
-0
app/entities_joiner/entities_joiner.py
app/entities_joiner/entities_joiner.py
+52
-0
app/entities_joiner/ids_loader.py
app/entities_joiner/ids_loader.py
+38
-0
app/entities_joiner/standardisation.py
app/entities_joiner/standardisation.py
+40
-0
app/entities_joiner/test/__init__.py
app/entities_joiner/test/__init__.py
+0
-0
app/entities_joiner/test/test_entities_joiner.py
app/entities_joiner/test/test_entities_joiner.py
+104
-0
app/entities_joiner/test/test_ids_loader.py
app/entities_joiner/test/test_ids_loader.py
+121
-0
app/swagger/swagger.yaml
app/swagger/swagger.yaml
+2
-2
functional_tests/specific_tests/fun_test_entities_join_0.py
functional_tests/specific_tests/fun_test_entities_join_0.py
+26
-0
functional_tests/specific_tests/fun_test_get_context_data.py
functional_tests/specific_tests/fun_test_get_context_data.py
+1
-1
No files found.
app/blueprints/entities_join/controllers/entities_join_controller.py
View file @
23b2075a
...
...
@@ -40,4 +40,7 @@ def get_link_to_related_items():
return
jsonify
(
json_response
)
except
entities_join_service
.
EntitiesJoinServiceError
as
error
:
abort
(
500
,
msg
=
f
'Internal server error:
{
str
(
error
)
}
'
)
print
(
'error!'
)
print
(
error
)
abort
(
500
,
f
'Internal server error:
{
str
(
error
)
}
'
)
app/blueprints/entities_join/services/entities_join_service.py
View file @
23b2075a
"""
Service that handles the requests to the entities join
"""
import
json
from
app
import
app_logging
from
app.es_data
import
es_data
from
app.context_loader
import
context_loader
from
app.config
import
RUN_CONFIG
from
app.es_data
import
es_mappings
from
app.entities_joiner
import
entities_joiner
class
EntitiesJoinServiceError
(
Exception
):
...
...
@@ -23,7 +17,17 @@ def get_link_to_related_items(destination_entity_browser_state_template,
:param entity_to: destination entity of the join
:param es_query: query in elasticsearch for the dataset
:param selection_description: stringifyed javascript object describing de selection of items in the dataset
:return: a dict with the tiny url to the link with the generated state
"""
return
{
'msg'
:
'hola'
}
try
:
tiny_hash
=
entities_joiner
.
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
)
return
{
'tiny_hash'
:
tiny_hash
}
except
entities_joiner
.
EntitiesJoinerError
as
error
:
raise
EntitiesJoinServiceError
(
error
)
app/entities_joiner/__init__.py
0 → 100644
View file @
23b2075a
app/entities_joiner/entities_joiner.py
0 → 100644
View file @
23b2075a
"""
Module with functions to make joins of entities
"""
import
json
from
app.entities_joiner
import
standardisation
class
EntitiesJoinerError
(
Exception
):
"""Base class for exceptions in this file."""
def
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param entity_from: source entity of the items
:param entity_to: destination entity of the join
:param es_query: query in elasticsearch for the dataset
:param selection_description: stringifyed javascript object describing de selection of items in the dataset
:return: the hash to the link with the generated state
"""
parsed_from
=
None
try
:
parsed_from
=
standardisation
.
PossibleFroms
(
entity_from
)
except
ValueError
as
error
:
raise
EntitiesJoinerError
(
f
'entity_from:
{
str
(
error
)
}
. Possible values are
{
[
item
.
value
for
item
in
standardisation
.
PossibleFroms
]
}
'
)
parsed_to
=
None
try
:
parsed_to
=
standardisation
.
PossibleTos
(
entity_to
)
except
ValueError
as
error
:
raise
EntitiesJoinerError
(
f
'entity_to:
{
str
(
error
)
}
. Possible values are
{
[
item
.
value
for
item
in
standardisation
.
PossibleTos
]
}
'
)
selection_description_dict
=
json
.
loads
(
selection_description
)
try
:
parsed_selection_mode
=
standardisation
.
SelectionModes
(
selection_description_dict
[
'selectionMode'
])
if
parsed_selection_mode
==
standardisation
.
SelectionModes
.
NO_ITEMS_EXCEPT
:
if
len
(
selection_description_dict
.
get
(
'exceptions'
,
[]))
==
0
:
raise
EntitiesJoinerError
(
f
'When selection mode is
{
parsed_selection_mode
}
there must be at least one exception'
)
except
ValueError
as
error
:
raise
EntitiesJoinerError
(
f
'selectionMode:
{
str
(
error
)
}
. Possible values are
{
[
item
.
value
for
item
in
standardisation
.
SelectionModes
]
}
'
)
if
entity_to
==
entity_from
:
raise
EntitiesJoinerError
(
f
'entity_to (
{
entity_to
}
) and entity_from (
{
entity_from
}
) cannot be the same!'
)
return
'holaaaa'
app/entities_joiner/ids_loader.py
0 → 100644
View file @
23b2075a
"""
Module with functions that help to load the ids for the entities joiner
"""
from
app.entities_joiner
import
standardisation
def
get_ids_query
(
es_query
,
selection_description
,
from_property
):
"""
:param es_query: query for the dataset
:param selection_description: dict describing the selection
:param from_property: property to get to to the join
:return: the query to use to get the ids depending on the selection description
"""
selection_mode
=
selection_description
[
'selectionMode'
]
parsed_selection_mode
=
standardisation
.
SelectionModes
(
selection_mode
)
exceptions
=
selection_description
.
get
(
'exceptions'
,
[])
if
parsed_selection_mode
==
standardisation
.
SelectionModes
.
ALL_ITEMS_EXCEPT
:
if
len
(
exceptions
)
==
0
:
return
{
'query'
:
es_query
.
get
(
'query'
),
"_source"
:
[
from_property
],
}
else
:
dataset_query
=
es_query
.
get
(
'query'
)
if
dataset_query
.
get
(
'bool'
)
is
None
:
dataset_query
[
'bool'
]
=
{}
if
dataset_query
.
get
(
'bool'
).
get
(
'must_not'
)
is
None
:
dataset_query
[
'bool'
][
'must_not'
]
=
[]
dataset_query
[
'bool'
][
'must_not'
].
append
({
'terms'
:
{
from_property
:
exceptions
}
})
return
{
'query'
:
dataset_query
,
"_source"
:
[
from_property
],
}
app/entities_joiner/standardisation.py
0 → 100644
View file @
23b2075a
"""
Module that helps with the standardisation of the package
"""
from
enum
import
Enum
class
PossibleFroms
(
Enum
):
"""
Enumeration with the possible froms allowed
"""
CHEMBL_ACTIVITIES
=
'CHEMBL_ACTIVITIES'
CHEMBL_COMPOUNDS
=
'CHEMBL_COMPOUNDS'
CHEMBL_TARGETS
=
'CHEMBL_TARGETS'
CHEMBL_ASSAYS
=
'CHEMBL_ASSAYS'
CHEMBL_DOCUMENTS
=
'CHEMBL_DOCUMENTS'
CHEMBL_CELL_LINES
=
'CHEMBL_CELL_LINES'
CHEMBL_TISSUES
=
'CHEMBL_TISSUES'
CHEMBL_DRUG_WARNINGS
=
'CHEMBL_DRUG_WARNINGS'
class
PossibleTos
(
Enum
):
"""
Enumeration with the possible to allowed
"""
CHEMBL_ACTIVITIES
=
'CHEMBL_ACTIVITIES'
CHEMBL_COMPOUNDS
=
'CHEMBL_COMPOUNDS'
CHEMBL_TARGETS
=
'CHEMBL_TARGETS'
CHEMBL_ASSAYS
=
'CHEMBL_ASSAYS'
CHEMBL_DOCUMENTS
=
'CHEMBL_DOCUMENTS'
CHEMBL_CELL_LINES
=
'CHEMBL_CELL_LINES'
CHEMBL_TISSUES
=
'CHEMBL_TISSUES'
CHEMBL_DRUG_WARNINGS
=
'CHEMBL_DRUG_WARNINGS'
class
SelectionModes
(
Enum
):
"""
Possible selection modes
"""
ALL_ITEMS_EXCEPT
=
'allItemsExcept'
NO_ITEMS_EXCEPT
=
'noItemsExcept'
app/entities_joiner/test/__init__.py
0 → 100644
View file @
23b2075a
app/entities_joiner/test/test_entities_joiner.py
0 → 100644
View file @
23b2075a
"""
Module to test the entities joiner module
"""
import
unittest
from
app.entities_joiner
import
entities_joiner
class
TestEntitiesJoiner
(
unittest
.
TestCase
):
"""
Class to test the entities joiner module
"""
def
test_fails_when_from_and_to_are_equal
(
self
):
"""
Tests that it fails when from and to are equal
"""
destination_entity_browser_state_template
=
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state'
\
'/<GENERATED_STATE> '
entity_from
=
'CHEMBL_COMPOUNDS'
entity_to
=
'CHEMBL_COMPOUNDS'
es_query
=
'{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, '
\
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description
=
'{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", '
\
'"CHEMBL64"]} '
with
self
.
assertRaises
(
entities_joiner
.
EntitiesJoinerError
):
entities_joiner
.
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
)
def
test_fails_when_from_is_not_on_the_possibilities
(
self
):
"""
Tests that it fails when from is not on the possibilities
"""
destination_entity_browser_state_template
=
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state'
\
'/<GENERATED_STATE> '
entity_from
=
'NON_EXISTENT'
entity_to
=
'CHEMBL_COMPOUNDS'
es_query
=
'{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, '
\
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description
=
'{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", '
\
'"CHEMBL64"]} '
with
self
.
assertRaises
(
entities_joiner
.
EntitiesJoinerError
):
entities_joiner
.
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
)
def
test_fails_when_to_is_not_on_the_possibilities
(
self
):
"""
Tests that it fails when to is not on the possibilities
"""
destination_entity_browser_state_template
=
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state'
\
'/<GENERATED_STATE> '
entity_from
=
'CHEMBL_COMPOUNDS'
entity_to
=
'NON_EXISTENT'
es_query
=
'{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, '
\
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description
=
'{"selectionMode": "allItemsExcept","exceptions": ["CHEMBL3989861", "CHEMBL3989724", '
\
'"CHEMBL64"]} '
with
self
.
assertRaises
(
entities_joiner
.
EntitiesJoinerError
):
entities_joiner
.
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
)
def
test_fails_when_selection_mode_not_in_possibilities
(
self
):
"""
Tests that it fails when to is not on the possibilities
"""
destination_entity_browser_state_template
=
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state'
\
'/<GENERATED_STATE> '
entity_from
=
'CHEMBL_COMPOUNDS'
entity_to
=
'CHEMBL_TARGETS'
es_query
=
'{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, '
\
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description
=
'{"selectionMode": "non_existent","exceptions": ["CHEMBL3989861", "CHEMBL3989724", '
\
'"CHEMBL64"]} '
with
self
.
assertRaises
(
entities_joiner
.
EntitiesJoinerError
):
entities_joiner
.
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
)
def
test_fails_when_selecting_no_items
(
self
):
"""
Tests that it fails when no items are selected
"""
destination_entity_browser_state_template
=
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state'
\
'/<GENERATED_STATE> '
entity_from
=
'CHEMBL_COMPOUNDS'
entity_to
=
'CHEMBL_TARGETS'
es_query
=
'{"size": 24, "from": 0, "query": {"bool": {"must": [{"query_string": {"analyze_wildcard": True, '
\
'"query": "*"}}], "filter": []}}, "sort": []} '
selection_description
=
'{"selectionMode": "noItemsExcept","exceptions": []} '
with
self
.
assertRaises
(
entities_joiner
.
EntitiesJoinerError
):
entities_joiner
.
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
)
app/entities_joiner/test/test_ids_loader.py
0 → 100644
View file @
23b2075a
"""
Module to test the ids loader
"""
import
unittest
from
app.entities_joiner
import
ids_loader
class
TestIDsLoader
(
unittest
.
TestCase
):
"""
Class to test the ids loader module
"""
def
test_generates_query_to_get_ids_when_selecting_all
(
self
):
"""
test that it generates the required query when selected all items in dataset
"""
es_query
=
{
"query"
:
{
"bool"
:
{
"filter"
:
[],
"should"
:
[],
"must_not"
:
[]
}
},
"track_total_hits"
:
True
,
"size"
:
20
,
"from"
:
0
,
"_source"
:
[
"drug_warning.molecule_chembl_id"
,
"drug_warning.parent_molecule_chembl_id"
,
"drug_warning.warning_type"
,
"drug_warning.warning_class"
,
"drug_warning.warning_description"
,
"drug_warning.where"
,
"drug_warning.warning_refs"
],
"sort"
:
[]
}
selection_description
=
{
"selectionMode"
:
"allItemsExcept"
,
"exceptions"
:
[]
}
from_property
=
'drug_warning.molecule_chembl_id'
ids_query_must_be
=
{
"query"
:
{
"bool"
:
{
"filter"
:
[],
"should"
:
[],
"must_not"
:
[]
}
},
"_source"
:
[
from_property
],
}
ids_query_got
=
ids_loader
.
get_ids_query
(
es_query
,
selection_description
,
from_property
)
self
.
assertDictEqual
(
ids_query_must_be
,
ids_query_got
,
msg
=
'The query was not generated correctly when selecting all items!'
)
def
test_generates_query_to_get_ids_when_selecting_all_except_some
(
self
):
"""
test that it generates the required query when selected all items in dataset
"""
es_query
=
{
"query"
:
{
"bool"
:
{
"filter"
:
[],
"should"
:
[],
"must_not"
:
[]
}
},
"track_total_hits"
:
True
,
"size"
:
20
,
"from"
:
0
,
"_source"
:
[
"drug_warning.molecule_chembl_id"
,
"drug_warning.parent_molecule_chembl_id"
,
"drug_warning.warning_type"
,
"drug_warning.warning_class"
,
"drug_warning.warning_description"
,
"drug_warning.where"
,
"drug_warning.warning_refs"
],
"sort"
:
[]
}
selection_description
=
{
"selectionMode"
:
"allItemsExcept"
,
"exceptions"
:
[
'CHEMBL3989861'
,
'CHEMBL3989724'
,
'CHEMBL64'
]
}
from_property
=
'drug_warning.molecule_chembl_id'
ids_query_must_be
=
{
"query"
:
{
"bool"
:
{
"filter"
:
[],
"should"
:
[],
"must_not"
:
[
{
"terms"
:
{
from_property
:
selection_description
[
'exceptions'
]
}
}
]
}
},
"_source"
:
[
from_property
],
}
ids_query_got
=
ids_loader
.
get_ids_query
(
es_query
,
selection_description
,
from_property
)
self
.
assertDictEqual
(
ids_query_must_be
,
ids_query_got
,
msg
=
'The query was not generated correctly when selecting all items except some!'
)
def
test_generates_query_to_get_ids_when_selecting_none
(
self
):
"""
test that it generates the required query when select in dataset
"""
app/swagger/swagger.yaml
View file @
23b2075a
...
...
@@ -515,14 +515,14 @@ paths:
description
:
'
Source
entity
for
the
join.'
required
:
true
type
:
'
string'
enum
:
[
'
CHEMBL_COMPOUNDS'
,
'
CHEMBL_TARGETS'
,
'
CHEMBL_ASSAYS'
,
'
CHEMBL_DOCUMENTS'
,
'
CHEMBL_CELL_LINES'
,
'
CHEMBL_TISSUES'
]
enum
:
[
'
CHEMBL_ACTIVITIES'
,
'
CHEMBL_COMPOUNDS'
,
'
CHEMBL_TARGETS'
,
'
CHEMBL_ASSAYS'
,
'
CHEMBL_DOCUMENTS'
,
'
CHEMBL_CELL_LINES'
,
'
CHEMBL_TISSUES'
,
'
CHEMBL_DRUG_WARNINGS'
]
default
:
'
CHEMBL_COMPOUNDS'
-
name
:
'
entity_to'
in
:
'
formData'
description
:
'
The
entity
in
ChEMBL
to
do
the
join
with.'
required
:
true
type
:
'
string'
enum
:
[
'
CHEMBL_COMPOUNDS'
,
'
CHEMBL_TARGETS'
,
'
CHEMBL_ASSAYS'
,
'
CHEMBL_DOCUMENTS'
,
'
CHEMBL_CELL_LINES'
,
'
CHEMBL_TISSUES'
]
enum
:
[
'
CHEMBL_ACTIVITIES'
,
'
CHEMBL_COMPOUNDS'
,
'
CHEMBL_TARGETS'
,
'
CHEMBL_ASSAYS'
,
'
CHEMBL_DOCUMENTS'
,
'
CHEMBL_CELL_LINES'
,
'
CHEMBL_TISSUES'
,
'
CHEMBL_DRUG_WARNINGS'
]
default
:
'
CHEMBL_COMPOUNDS'
-
name
:
'
es_query'
in
:
'
formData'
...
...
functional_tests/specific_tests/fun_test_entities_join_0.py
0 → 100644
View file @
23b2075a
# pylint: disable=import-error
"""
Module that tests the endpoints to do joins among entities
"""
import
requests
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
"""
Tests doing a join among different entities
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print
(
'-------------------------------------------'
)
print
(
'Testing joins among entities'
)
print
(
'-------------------------------------------'
)
join_params
=
{
'destination_entity_browser_state_template'
:
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>'
,
'entity_from'
:
'CHEMBL_DRUG_WARNINGS'
,
'entity_to'
:
'CHEMBL_ACTIVITIES'
,
'es_query'
:
''
,
}
url
=
f
'
{
server_base_url
}
/entities_join/get_link_to_related_items'
functional_tests/specific_tests/fun_test_get_context_data.py
View file @
23b2075a
...
...
@@ -9,7 +9,7 @@ from specific_tests import utils
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
"""
Tests that getting the data of a context
Tests that getting the data of a context
works
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment