Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
ChEMBL
C
ChEMBL
Main Web Interface
Elasticsearch Proxy API
Commits
4b9b286d
Commit
4b9b286d
authored
Jun 29, 2021
by
David Mendez
Browse files
Entities Join: allow to include previous hash to encode it in state
parent
b438b304
Changes
12
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
90 additions
and
35 deletions
+90
-35
app/blueprints/entities_join/controllers/entities_join_controller.py
...nts/entities_join/controllers/entities_join_controller.py
+3
-2
app/blueprints/entities_join/controllers/marshmallow_schemas.py
...ueprints/entities_join/controllers/marshmallow_schemas.py
+1
-0
app/blueprints/entities_join/services/entities_join_service.py
...lueprints/entities_join/services/entities_join_service.py
+5
-4
app/entities_joiner/entities_joiner.py
app/entities_joiner/entities_joiner.py
+11
-21
app/entities_joiner/standardisation.py
app/entities_joiner/standardisation.py
+6
-0
app/swagger/swagger.yaml
app/swagger/swagger.yaml
+5
-0
functional_tests/specific_tests/entities_join/fun_test_entities_join_0.py
.../specific_tests/entities_join/fun_test_entities_join_0.py
+2
-1
functional_tests/specific_tests/entities_join/fun_test_entities_join_1.py
.../specific_tests/entities_join/fun_test_entities_join_1.py
+3
-1
functional_tests/specific_tests/entities_join/fun_test_entities_join_2.py
.../specific_tests/entities_join/fun_test_entities_join_2.py
+22
-0
functional_tests/specific_tests/entities_join/fun_test_entities_join_3.py
.../specific_tests/entities_join/fun_test_entities_join_3.py
+22
-0
functional_tests/specific_tests/entities_join/fun_test_entities_join_all_tests.py
...c_tests/entities_join/fun_test_entities_join_all_tests.py
+4
-2
functional_tests/specific_tests/entities_join/utils.py
functional_tests/specific_tests/entities_join/utils.py
+6
-4
No files found.
app/blueprints/entities_join/controllers/entities_join_controller.py
View file @
4b9b286d
...
...
@@ -26,6 +26,7 @@ def get_link_to_related_items():
entity_to
=
request_parameters
.
sanitise_parameter
(
form_data
.
get
(
'entity_to'
))
es_query
=
request_parameters
.
sanitise_parameter
(
form_data
.
get
(
'es_query'
))
selection_description
=
request_parameters
.
sanitise_parameter
(
form_data
.
get
(
'selection_description'
))
previous_hash
=
request_parameters
.
sanitise_parameter
(
form_data
.
get
(
'previous_hash'
))
app_logging
.
debug
(
f
'destination_entity_browser_state_template:
{
destination_entity_browser_state_template
}
'
)
app_logging
.
debug
(
f
'entity_from:
{
entity_from
}
'
)
...
...
@@ -34,9 +35,9 @@ def get_link_to_related_items():
app_logging
.
debug
(
f
'selection_description:
{
selection_description
}
'
)
try
:
json_response
=
entities_join_service
.
get_
l
in
k
_to_related_items
(
destination_entity_browser_state_template
,
json_response
=
entities_join_service
.
get_
t
in
y_hash
_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
)
selection_description
,
previous_hash
)
return
jsonify
(
json_response
)
except
entities_join_service
.
EntitiesJoinServiceError
as
error
:
...
...
app/blueprints/entities_join/controllers/marshmallow_schemas.py
View file @
4b9b286d
...
...
@@ -13,3 +13,4 @@ class EntitiesJoinQuery(Schema):
entity_to
=
fields
.
String
(
required
=
True
)
es_query
=
fields
.
String
(
required
=
True
)
selection_description
=
fields
.
String
(
required
=
True
)
previous_hash
=
fields
.
String
()
app/blueprints/entities_join/services/entities_join_service.py
View file @
4b9b286d
...
...
@@ -8,22 +8,23 @@ class EntitiesJoinServiceError(Exception):
"""Base class for exceptions in this file."""
def
get_
l
in
k
_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
):
def
get_
t
in
y_hash
_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
,
previous_hash
):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param entity_from: source entity of the items
:param entity_to: destination entity of the join
:param es_query: query in elasticsearch for the dataset
:param selection_description: stringifyed javascript object describing de selection of items in the dataset
:param previous_hash: hash of the state that originated this join of entities
:return: a dict with the tiny url to the link with the generated state
"""
try
:
tiny_hash
=
entities_joiner
.
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
es_query
,
selection_description
)
selection_description
,
previous_hash
)
return
{
'tiny_hash'
:
tiny_hash
...
...
app/entities_joiner/entities_joiner.py
View file @
4b9b286d
...
...
@@ -15,13 +15,14 @@ class EntitiesJoinerError(Exception):
def
get_tiny_hash_to_related_items
(
destination_entity_browser_state_template
,
entity_from
,
entity_to
,
raw_es_query
,
raw_selection_description
):
raw_selection_description
,
previous_hash
):
"""
:param destination_entity_browser_state_template: template for building the resulting browser url
:param entity_from: source entity of the items
:param entity_to: destination entity of the join
:param raw_es_query: stringifyed query in elasticsearch for the dataset
:param raw_selection_description: stringifyed javascript object describing de selection of items in the dataset
:param previous_hash: hash of the state that originated this join of entities
:return: the hash to the link with the generated state
"""
parsed_from_entity
=
None
...
...
@@ -52,16 +53,12 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
if
entity_to
==
entity_from
:
raise
EntitiesJoinerError
(
f
'entity_to (
{
entity_to
}
) and entity_from (
{
entity_from
}
) cannot be the same!'
)
print
(
'load ids:'
)
index_name
=
standardisation
.
get_index_name_for_from_entity
(
parsed_from_entity
)
fail_if_null
(
index_name
,
'index name'
,
parsed_from_entity
,
parsed_to_entity
)
print
(
'index_name: '
,
index_name
)
from_property
=
standardisation
.
get_from_property
(
parsed_from_entity
,
parsed_to_entity
)
fail_if_null
(
from_property
,
'from property'
,
parsed_from_entity
,
parsed_to_entity
)
print
(
'from_property: '
,
from_property
)
selection_description
=
json
.
loads
(
raw_selection_description
)
es_query
=
json
.
loads
(
raw_es_query
)
ids
=
ids_loader
.
load_ids_for_query
(
es_query
,
selection_description
,
from_property
,
index_name
)
...
...
@@ -76,9 +73,8 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
fail_if_null
(
browser_name
,
'browser name'
,
parsed_from_entity
,
parsed_to_entity
)
join_state_hash
=
get_join_state_hash
(
ids
,
to_property
,
settings_path
,
browser_name
,
destination_entity_browser_state_template
)
print
(
'join_state_hash: '
)
print
(
join_state_hash
)
destination_entity_browser_state_template
,
previous_hash
)
return
join_state_hash
...
...
@@ -96,23 +92,23 @@ def fail_if_null(value, value_name, parsed_from_entity, parsed_to_entity):
f
'to
{
parsed_to_entity
.
value
}
'
)
def
get_join_state_hash
(
ids
,
to_property
,
settings_path
,
browser_name
,
destination_entity_browser_state_template
):
def
get_join_state_hash
(
ids
,
to_property
,
settings_path
,
browser_name
,
destination_entity_browser_state_template
,
previous_hash
):
"""
:param ids: list of its for the join
:param to_property: entity to which to do the join
:param settings_path: settings path to be used by the user interface
:param browser_name: name of the destination browser used to build the desired url
:param destination_entity_browser_state_template: template to build the url for the destination entity browser
:param previous_hash: hash of the state that originated this join of entities
:return: the hash for the query state used for the join
"""
print
(
'get_join_query'
)
print
(
'ids: '
)
print
(
ids
)
print
(
'to_property: '
)
print
(
to_property
)
ids_clauses
=
" OR "
.
join
([
f
'"
{
item_id
}
"'
for
item_id
in
ids
])
desired_state
=
{
'origin'
:
{
'type'
:
'ENTITIES_JOIN'
,
'previous_hash'
:
previous_hash
,
},
'list'
:
{
'settings_path'
:
settings_path
,
'custom_query'
:
f
'
{
to_property
}
(
{
ids_clauses
}
)'
,
...
...
@@ -121,10 +117,7 @@ def get_join_state_hash(ids, to_property, settings_path, browser_name, destinati
'at_least_one_facet_is_selected'
:
False
,
}
}
print
(
'desired_state: '
,
desired_state
)
b64_desired_state
=
base64
.
b64encode
(
json
.
dumps
(
desired_state
).
encode
())
print
(
'b64_desired_state: '
,
b64_desired_state
)
print
(
'destination_entity_browser_state_template: '
,
destination_entity_browser_state_template
)
destination_entity_browser_state_url
=
destination_entity_browser_state_template
.
replace
(
'<BROWSER_NAME>'
,
browser_name
...
...
@@ -134,10 +127,7 @@ def get_join_state_hash(ids, to_property, settings_path, browser_name, destinati
b64_desired_state
.
decode
()
)
print
(
'destination_entity_browser_state_url: '
,
destination_entity_browser_state_url
)
hashable_part
=
f
'#
{
destination_entity_browser_state_url
.
split
(
"#"
)[
1
]
}
'
print
(
'hashable_part: '
,
hashable_part
)
return
get_destination_url_hash
(
hashable_part
)
...
...
app/entities_joiner/standardisation.py
View file @
4b9b286d
...
...
@@ -69,6 +69,10 @@ JOIN_PROPERTIES = {
PossibleEntitiesTo
.
CHEMBL_ACTIVITIES
:
{
'from_property'
:
'drug_warning.molecule_chembl_id'
,
'to_property'
:
'molecule_chembl_id'
},
PossibleEntitiesTo
.
CHEMBL_COMPOUNDS
:
{
'from_property'
:
'drug_warning.molecule_chembl_id'
,
'to_property'
:
'molecule_chembl_id'
}
}
}
...
...
@@ -78,10 +82,12 @@ JOIN_PROPERTIES = {
# This is used by the frontend to know which settings use to load the page, should be discarded in the future
SETTINGS_PATHS_FOR_TO_ENTITIES
=
{
PossibleEntitiesTo
.
CHEMBL_ACTIVITIES
:
'ES_INDEXES_NO_MAIN_SEARCH.ACTIVITY'
,
PossibleEntitiesTo
.
CHEMBL_COMPOUNDS
:
'ES_INDEXES_NO_MAIN_SEARCH.COMPOUND_COOL_CARDS'
}
BROWSER_NAMES_FOR_TO_ENTITIES
=
{
PossibleEntitiesTo
.
CHEMBL_ACTIVITIES
:
'activities'
,
PossibleEntitiesTo
.
CHEMBL_COMPOUNDS
:
'compounds'
}
...
...
app/swagger/swagger.yaml
View file @
4b9b286d
...
...
@@ -535,6 +535,11 @@ paths:
description
:
'
stringifyied
version
of
the
description
of
the
selection
parameters
of
the
dataset.
//selectionMode
options:
["allItemsExcept",
"noItemsExcept"]'
type
:
'
string'
default
:
'
{"selectionMode":
"allItemsExcept","exceptions":
["CHEMBL3989861",
"CHEMBL3989724",
"CHEMBL64"]}'
-
name
:
'
previous_hash'
in
:
'
formData'
description
:
'
hash
of
the
state
that
is
making
the
call'
type
:
'
string'
default
:
'
sQseUMn43BEG1hQ-doPggw=='
responses
:
'
200'
:
description
:
"
success"
...
...
functional_tests/specific_tests/entities_join/fun_test_entities_join_0.py
View file @
4b9b286d
...
...
@@ -17,4 +17,5 @@ def run_test(server_base_url, delayed_jobs_server_base_path):
dataset_query
=
utils
.
load_json_data
(
'functional_tests/specific_tests/data/entities_join_query_0.json'
)
selection_description
=
{
"selectionMode"
:
"allItemsExcept"
,
"exceptions"
:
[]}
entities_join_utils
.
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
)
entities_join_utils
.
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
,
entity_from
=
'CHEMBL_DRUG_WARNINGS'
,
entity_to
=
'CHEMBL_ACTIVITIES'
)
functional_tests/specific_tests/entities_join/fun_test_entities_join_1.py
View file @
4b9b286d
...
...
@@ -18,4 +18,6 @@ def run_test(server_base_url, delayed_jobs_server_base_path):
print
(
'-------------------------------------------'
)
dataset_query
=
utils
.
load_json_data
(
'functional_tests/specific_tests/data/entities_join_query_0.json'
)
selection_description
=
{
"selectionMode"
:
"allItemsExcept"
,
"exceptions"
:
[
'CHEMBL2107495'
,
'CHEMBL340978'
]}
entities_join_utils
.
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
)
entities_join_utils
.
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
,
entity_from
=
'CHEMBL_DRUG_WARNINGS'
,
entity_to
=
'CHEMBL_ACTIVITIES'
)
functional_tests/specific_tests/entities_join/fun_test_entities_join_2.py
0 → 100644
View file @
4b9b286d
# pylint: disable=import-error
"""
Module that tests the endpoints to do joins among entities selecting no ids except some
"""
from
specific_tests
import
utils
from
specific_tests.entities_join
import
utils
as
entities_join_utils
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
"""
Tests doing a join among different entities selecting no ids except some
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print
(
'-------------------------------------------'
)
print
(
'Testing joins among entities selecting all none except'
)
print
(
'-------------------------------------------'
)
dataset_query
=
utils
.
load_json_data
(
'functional_tests/specific_tests/data/entities_join_query_0.json'
)
selection_description
=
{
"selectionMode"
:
"noItemsExcept"
,
"exceptions"
:
[
'CHEMBL2107495'
,
'CHEMBL340978'
]}
entities_join_utils
.
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
,
entity_from
=
'CHEMBL_DRUG_WARNINGS'
,
entity_to
=
'CHEMBL_ACTIVITIES'
)
functional_tests/specific_tests/entities_join/fun_test_entities_join_3.py
0 → 100644
View file @
4b9b286d
# pylint: disable=import-error
"""
Module that tests the endpoints to do joins among entities selecting no ids except some
"""
from
specific_tests
import
utils
from
specific_tests.entities_join
import
utils
as
entities_join_utils
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
"""
Tests doing a join among different entities selecting no ids except some
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print
(
'-------------------------------------------'
)
print
(
'Testing joins among entities selecting all none except'
)
print
(
'-------------------------------------------'
)
dataset_query
=
utils
.
load_json_data
(
'functional_tests/specific_tests/data/entities_join_query_0.json'
)
selection_description
=
{
"selectionMode"
:
"allItemsExcept"
,
"exceptions"
:
[]}
entities_join_utils
.
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
,
entity_from
=
'CHEMBL_DRUG_WARNINGS'
,
entity_to
=
'CHEMBL_COMPOUNDS'
)
functional_tests/specific_tests/entities_join/fun_test_entities_join_all_tests.py
View file @
4b9b286d
"""
Module that runs all the tests related to the entities join
"""
from
specific_tests.entities_join
import
fun_test_entities_join_0
,
fun_test_entities_join_1
from
specific_tests.entities_join
import
fun_test_entities_join_0
,
fun_test_entities_join_1
,
fun_test_entities_join_2
,
\
fun_test_entities_join_3
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
...
...
@@ -10,5 +11,6 @@ def run_test(server_base_url, delayed_jobs_server_base_path):
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
for
test_module
in
[
fun_test_entities_join_0
,
fun_test_entities_join_1
]:
for
test_module
in
[
fun_test_entities_join_0
,
fun_test_entities_join_1
,
fun_test_entities_join_2
,
fun_test_entities_join_3
]:
test_module
.
run_test
(
server_base_url
,
delayed_jobs_server_base_path
)
functional_tests/specific_tests/entities_join/utils.py
View file @
4b9b286d
...
...
@@ -7,21 +7,23 @@ import requests
from
specific_tests
import
utils
def
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
):
def
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
,
entity_from
,
entity_to
):
"""
Performs the testing of the entities join endpoint given the parameters
:param dataset_query: query of the current dataset
:param selection_description: dict describing the selection
:param server_base_url: base url of the server being used for the tests
:param entity_from: entity from which to do the join
"""
join_params
=
{
'destination_entity_browser_state_template'
:
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>'
,
'entity_from'
:
'CHEMBL_DRUG_WARNINGS'
,
'entity_to'
:
'CHEMBL_ACTIVITIES'
,
'entity_from'
:
entity_from
,
'entity_to'
:
entity_to
,
'es_query'
:
json
.
dumps
(
dataset_query
),
'selection_description'
:
json
.
dumps
(
selection_description
)
'selection_description'
:
json
.
dumps
(
selection_description
),
'previous_hash'
:
'sQseUMn43BEG1hQ-doPggw=='
}
url
=
f
'
{
server_base_url
}
/entities_join/get_link_to_related_items'
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment