Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
ChEMBL
C
ChEMBL
Main Web Interface
Elasticsearch Proxy API
Commits
b438b304
Commit
b438b304
authored
Jun 29, 2021
by
David Mendez
Browse files
Entities join endpoint: add more functional tests
parent
0f026994
Changes
14
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
266 additions
and
112 deletions
+266
-112
.gitignore
.gitignore
+1
-0
app/entities_joiner/entities_joiner.py
app/entities_joiner/entities_joiner.py
+71
-18
app/entities_joiner/standardisation.py
app/entities_joiner/standardisation.py
+24
-0
app/url_shortening/url_shortener.py
app/url_shortening/url_shortener.py
+3
-1
functional_tests/run_functional_tests.py
functional_tests/run_functional_tests.py
+5
-2
functional_tests/specific_tests/data/entities_join_query_0.json
...onal_tests/specific_tests/data/entities_join_query_0.json
+49
-0
functional_tests/specific_tests/entities_join/__init__.py
functional_tests/specific_tests/entities_join/__init__.py
+0
-0
functional_tests/specific_tests/entities_join/fun_test_entities_join_0.py
.../specific_tests/entities_join/fun_test_entities_join_0.py
+20
-0
functional_tests/specific_tests/entities_join/fun_test_entities_join_1.py
.../specific_tests/entities_join/fun_test_entities_join_1.py
+21
-0
functional_tests/specific_tests/entities_join/fun_test_entities_join_all_tests.py
...c_tests/entities_join/fun_test_entities_join_all_tests.py
+14
-0
functional_tests/specific_tests/entities_join/utils.py
functional_tests/specific_tests/entities_join/utils.py
+47
-0
functional_tests/specific_tests/fun_test_entities_join_0.py
functional_tests/specific_tests/fun_test_entities_join_0.py
+0
-90
functional_tests/specific_tests/fun_test_url_shortening.py
functional_tests/specific_tests/fun_test_url_shortening.py
+1
-1
functional_tests/specific_tests/utils.py
functional_tests/specific_tests/utils.py
+10
-0
No files found.
.gitignore
View file @
b438b304
...
...
@@ -139,4 +139,5 @@ app/static/
output.csv
deleteme
deleteme.json
\ No newline at end of file
app/entities_joiner/entities_joiner.py
View file @
b438b304
...
...
@@ -2,9 +2,11 @@
Module with functions to make joins of entities
"""
import
json
import
base64
from
app.entities_joiner
import
standardisation
from
app.entities_joiner
import
ids_loader
from
app.url_shortening
import
url_shortener
class
EntitiesJoinerError
(
Exception
):
...
...
@@ -53,15 +55,11 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
print
(
'load ids:'
)
index_name
=
standardisation
.
get_index_name_for_from_entity
(
parsed_from_entity
)
if
index_name
is
None
:
raise
EntitiesJoinerError
(
f
'There is no index name configured for queries from
{
parsed_from_entity
.
value
}
'
)
fail_if_null
(
index_name
,
'index name'
,
parsed_from_entity
,
parsed_to_entity
)
print
(
'index_name: '
,
index_name
)
from_property
=
standardisation
.
get_from_property
(
parsed_from_entity
,
parsed_to_entity
)
if
from_property
is
None
:
raise
EntitiesJoinerError
(
f
'There is no from property configured for queries from
{
parsed_from_entity
.
value
}
'
)
fail_if_null
(
from_property
,
'from property'
,
parsed_from_entity
,
parsed_to_entity
)
print
(
'from_property: '
,
from_property
)
selection_description
=
json
.
loads
(
raw_selection_description
)
...
...
@@ -69,18 +67,43 @@ def get_tiny_hash_to_related_items(destination_entity_browser_state_template,
ids
=
ids_loader
.
load_ids_for_query
(
es_query
,
selection_description
,
from_property
,
index_name
)
to_property
=
standardisation
.
get_to_property
(
parsed_from_entity
,
parsed_to_entity
)
join_query
=
get_join_query
(
ids
,
to_property
)
print
(
'join_query: '
)
print
(
join_query
)
fail_if_null
(
to_property
,
'to property'
,
parsed_from_entity
,
parsed_to_entity
)
settings_path
=
standardisation
.
get_settings_path_for_to_entity
(
parsed_to_entity
)
fail_if_null
(
settings_path
,
'settings path'
,
parsed_from_entity
,
parsed_to_entity
)
browser_name
=
standardisation
.
get_browser_names_for_to_entity
(
parsed_to_entity
)
fail_if_null
(
browser_name
,
'browser name'
,
parsed_from_entity
,
parsed_to_entity
)
join_state_hash
=
get_join_state_hash
(
ids
,
to_property
,
settings_path
,
browser_name
,
destination_entity_browser_state_template
)
print
(
'join_state_hash: '
)
print
(
join_state_hash
)
return
join_state_hash
return
'holaaaa'
def
fail_if_null
(
value
,
value_name
,
parsed_from_entity
,
parsed_to_entity
):
"""
Fails if the value is null and uses the rest of the parameters to build an error message
:param value: value to check
:param value_name: name of the value for the error message
:param parsed_from_entity: parsed from entity
:param parsed_to_entity: parsed to entity
"""
if
value
is
None
:
raise
EntitiesJoinerError
(
f
'There is no
{
value_name
}
configured for queries from
{
parsed_from_entity
.
value
}
'
f
'to
{
parsed_to_entity
.
value
}
'
)
def
get_join_
query
(
ids
,
to_property
):
def
get_join_
state_hash
(
ids
,
to_property
,
settings_path
,
browser_name
,
destination_entity_browser_state_template
):
"""
:param ids: list of its for the join
:param to_property: entity to which to do the join
:return: the query to use for the join
:param settings_path: settings path to be used by the user interface
:param browser_name: name of the destination browser used to build the desired url
:param destination_entity_browser_state_template: template to build the url for the destination entity browser
:return: the hash for the query state used for the join
"""
print
(
'get_join_query'
)
print
(
'ids: '
)
...
...
@@ -88,11 +111,41 @@ def get_join_query(ids, to_property):
print
(
'to_property: '
)
print
(
to_property
)
return
{
"query"
:
{
"terms"
:
{
"molecule_chembl_id"
:
[
"CHEMBL2107495"
,
"CHEMBL1204165"
]
}
ids_clauses
=
" OR "
.
join
([
f
'"
{
item_id
}
"'
for
item_id
in
ids
])
desired_state
=
{
'list'
:
{
'settings_path'
:
settings_path
,
'custom_query'
:
f
'
{
to_property
}
(
{
ids_clauses
}
)'
,
'use_custom_query'
:
True
,
'search_term'
:
""
,
'at_least_one_facet_is_selected'
:
False
,
}
}
print
(
'desired_state: '
,
desired_state
)
b64_desired_state
=
base64
.
b64encode
(
json
.
dumps
(
desired_state
).
encode
())
print
(
'b64_desired_state: '
,
b64_desired_state
)
print
(
'destination_entity_browser_state_template: '
,
destination_entity_browser_state_template
)
destination_entity_browser_state_url
=
destination_entity_browser_state_template
.
replace
(
'<BROWSER_NAME>'
,
browser_name
)
destination_entity_browser_state_url
=
destination_entity_browser_state_url
.
replace
(
'<GENERATED_STATE>'
,
b64_desired_state
.
decode
()
)
print
(
'destination_entity_browser_state_url: '
,
destination_entity_browser_state_url
)
hashable_part
=
f
'#
{
destination_entity_browser_state_url
.
split
(
"#"
)[
1
]
}
'
print
(
'hashable_part: '
,
hashable_part
)
return
get_destination_url_hash
(
hashable_part
)
def
get_destination_url_hash
(
hashable_part
):
"""
:param hashable_part: of the destination url generated
:return: a hash generated using the hasable url part of the destination browser
"""
shortening_response
=
url_shortener
.
shorten_url
(
hashable_part
)
return
shortening_response
[
'hash'
]
app/entities_joiner/standardisation.py
View file @
b438b304
...
...
@@ -73,7 +73,15 @@ JOIN_PROPERTIES = {
}
}
}
}
# This is used by the frontend to know which settings use to load the page, should be discarded in the future
SETTINGS_PATHS_FOR_TO_ENTITIES
=
{
PossibleEntitiesTo
.
CHEMBL_ACTIVITIES
:
'ES_INDEXES_NO_MAIN_SEARCH.ACTIVITY'
,
}
BROWSER_NAMES_FOR_TO_ENTITIES
=
{
PossibleEntitiesTo
.
CHEMBL_ACTIVITIES
:
'activities'
,
}
...
...
@@ -101,3 +109,19 @@ def get_to_property(parsed_from_entity, parsed_to_entity):
get
(
parsed_from_entity
,
{}).
\
get
(
'to'
,
{}).
\
get
(
parsed_to_entity
,
{}).
get
(
'to_property'
)
def
get_settings_path_for_to_entity
(
parsed_to_entity
):
"""
:param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:return: the settings path used to create the destination page
"""
return
SETTINGS_PATHS_FOR_TO_ENTITIES
.
get
(
parsed_to_entity
)
def
get_browser_names_for_to_entity
(
parsed_to_entity
):
"""
:param parsed_to_entity: entity 'to' parsed by the PossibleEntitiesTo enum
:return: the browser name used in the url for the destination page
"""
return
BROWSER_NAMES_FOR_TO_ENTITIES
.
get
(
parsed_to_entity
)
app/url_shortening/url_shortener.py
View file @
b438b304
...
...
@@ -11,14 +11,16 @@ from app import app_logging
from
app.usage_statistics
import
statistics_saver
from
app.url_shortening.expired_urls_deletion
import
ExpiredURLsDeletionThread
class
URLNotFoundError
(
Exception
):
"""
Error raises when an url was not found
"""
# ----------------------------------------------------------------------------------------------------------------------
# URL Shortening
# ----------------------------------------------------------------------------------------------------------------------
def
shorten_url
(
long_url
):
"""
:param long_url:
...
...
functional_tests/run_functional_tests.py
View file @
b438b304
...
...
@@ -12,7 +12,10 @@ from specific_tests import fun_test_simple_query, fun_test_query_with_context, \
fun_test_go_slim_target_classification
,
fun_test_in_vivo_assay_classification
,
\
fun_test_organism_taxonomy_target_classification
,
fun_test_protein_target_classification
,
\
fun_test_covid_entities_records
,
fun_test_database_summary
,
fun_test_entities_records
,
\
fun_test_get_all_properties
,
fun_test_identify_separator
,
fun_test_entities_join_0
fun_test_get_all_properties
,
fun_test_identify_separator
from
specific_tests.entities_join
import
fun_test_entities_join_all_tests
PARSER
=
argparse
.
ArgumentParser
()
PARSER
.
add_argument
(
'server_base_path'
,
help
=
'server base path to run the tests against'
,
...
...
@@ -28,7 +31,7 @@ def run():
"""
print
(
f
'Running functional tests on
{
ARGS
.
server_base_path
}
'
)
for
test_module
in
[
fun_test_entities_join_
0
]:
for
test_module
in
[
fun_test_entities_join_
all_tests
]:
test_module
.
run_test
(
ARGS
.
server_base_path
,
ARGS
.
delayed_jobs_server_base_path
)
...
...
functional_tests/specific_tests/data/entities_join_query_0.json
0 → 100644
View file @
b438b304
{
"query"
:
{
"bool"
:
{
"filter"
:
[
{
"bool"
:
{
"should"
:
[
{
"terms"
:
{
"drug_warning.warning_class"
:
[
"Hepatotoxicity"
]
}
}
]
}
},
{
"bool"
:
{
"should"
:
[
{
"terms"
:
{
"drug_warning.where.country"
:
[
"United Kingdom"
]
}
}
]
}
}
],
"should"
:
[],
"must_not"
:
[]
}
},
"track_total_hits"
:
true
,
"size"
:
20
,
"from"
:
0
,
"_source"
:
[
"drug_warning.molecule_chembl_id"
,
"drug_warning.parent_molecule_chembl_id"
,
"drug_warning.warning_type"
,
"drug_warning.warning_class"
,
"drug_warning.warning_description"
,
"drug_warning.where"
,
"drug_warning.warning_refs"
],
"sort"
:
[]
}
\ No newline at end of file
functional_tests/specific_tests/entities_join/__init__.py
0 → 100644
View file @
b438b304
functional_tests/specific_tests/entities_join/fun_test_entities_join_0.py
0 → 100644
View file @
b438b304
"""
Module that tests the endpoints to do joins among entities selecting all ids
"""
from
specific_tests
import
utils
from
specific_tests.entities_join
import
utils
as
entities_join_utils
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
"""
Tests doing a join among different entities selecting all ids
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print
(
'-------------------------------------------'
)
print
(
'Testing joins among entities'
)
print
(
'-------------------------------------------'
)
dataset_query
=
utils
.
load_json_data
(
'functional_tests/specific_tests/data/entities_join_query_0.json'
)
selection_description
=
{
"selectionMode"
:
"allItemsExcept"
,
"exceptions"
:
[]}
entities_join_utils
.
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
)
functional_tests/specific_tests/entities_join/fun_test_entities_join_1.py
0 → 100644
View file @
b438b304
# pylint: disable=import-error
"""
Module that tests the endpoints to do joins among entities selecting all ids except some
"""
from
specific_tests
import
utils
from
specific_tests.entities_join
import
utils
as
entities_join_utils
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
"""
Tests doing a join among different entities selecting all ids except some
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print
(
'-------------------------------------------'
)
print
(
'Testing joins among entities selecting all except some'
)
print
(
'-------------------------------------------'
)
dataset_query
=
utils
.
load_json_data
(
'functional_tests/specific_tests/data/entities_join_query_0.json'
)
selection_description
=
{
"selectionMode"
:
"allItemsExcept"
,
"exceptions"
:
[
'CHEMBL2107495'
,
'CHEMBL340978'
]}
entities_join_utils
.
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
)
functional_tests/specific_tests/entities_join/fun_test_entities_join_all_tests.py
0 → 100644
View file @
b438b304
"""
Module that runs all the tests related to the entities join
"""
from
specific_tests.entities_join
import
fun_test_entities_join_0
,
fun_test_entities_join_1
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
"""
Tests doing a join among different entities selecting all ids
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
for
test_module
in
[
fun_test_entities_join_0
,
fun_test_entities_join_1
]:
test_module
.
run_test
(
server_base_url
,
delayed_jobs_server_base_path
)
functional_tests/specific_tests/entities_join/utils.py
0 → 100644
View file @
b438b304
"""
Module with utils for entities join tests
"""
import
json
import
requests
from
specific_tests
import
utils
def
test_entities_join
(
dataset_query
,
selection_description
,
server_base_url
):
"""
Performs the testing of the entities join endpoint given the parameters
:param dataset_query: query of the current dataset
:param selection_description: dict describing the selection
:param server_base_url: base url of the server being used for the tests
"""
join_params
=
{
'destination_entity_browser_state_template'
:
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>'
,
'entity_from'
:
'CHEMBL_DRUG_WARNINGS'
,
'entity_to'
:
'CHEMBL_ACTIVITIES'
,
'es_query'
:
json
.
dumps
(
dataset_query
),
'selection_description'
:
json
.
dumps
(
selection_description
)
}
url
=
f
'
{
server_base_url
}
/entities_join/get_link_to_related_items'
request
=
requests
.
post
(
url
,
data
=
join_params
)
status_code
=
request
.
status_code
print
(
f
'status_code:
{
status_code
}
'
)
response_text
=
request
.
text
utils
.
print_es_response
(
response_text
)
assert
status_code
==
200
,
'The request failed!'
url_hash_got
=
request
.
json
()[
'tiny_hash'
]
print
(
'url_hash_got: '
,
url_hash_got
)
expansion_url
=
f
'
{
server_base_url
}
/url_shortening/expand_url/
{
url_hash_got
}
'
print
(
f
'Checking tiny hash:
{
expansion_url
}
'
)
request
=
requests
.
get
(
expansion_url
)
status_code
=
request
.
status_code
print
(
f
'status_code:
{
status_code
}
'
)
response_text
=
request
.
text
utils
.
print_es_response
(
response_text
)
assert
status_code
==
200
,
'The request failed!'
functional_tests/specific_tests/fun_test_entities_join_0.py
deleted
100644 → 0
View file @
0f026994
# pylint: disable=import-error
"""
Module that tests the endpoints to do joins among entities selecting all ids
"""
import
json
import
requests
from
specific_tests
import
utils
def
run_test
(
server_base_url
,
delayed_jobs_server_base_path
):
"""
Tests doing a join among different entities selecting all ids
:param server_base_url: base url of the running server. E.g. http://127.0.0.1:5000
:param delayed_jobs_server_base_path: base path for the delayed_jobs
"""
print
(
'-------------------------------------------'
)
print
(
'Testing joins among entities'
)
print
(
'-------------------------------------------'
)
dataset_query
=
{
"query"
:
{
"bool"
:
{
"filter"
:
[
{
"bool"
:
{
"should"
:
[
{
"terms"
:
{
"drug_warning.warning_class"
:
[
"Hepatotoxicity"
]
}
}
]
}
},
{
"bool"
:
{
"should"
:
[
{
"terms"
:
{
"drug_warning.where.country"
:
[
"United Kingdom"
]
}
}
]
}
}
],
"should"
:
[],
"must_not"
:
[]
}
},
"track_total_hits"
:
True
,
"size"
:
20
,
"from"
:
0
,
"_source"
:
[
"drug_warning.molecule_chembl_id"
,
"drug_warning.parent_molecule_chembl_id"
,
"drug_warning.warning_type"
,
"drug_warning.warning_class"
,
"drug_warning.warning_description"
,
"drug_warning.where"
,
"drug_warning.warning_refs"
],
"sort"
:
[]
}
selection_description
=
{
"selectionMode"
:
"allItemsExcept"
,
"exceptions"
:
[]}
join_params
=
{
'destination_entity_browser_state_template'
:
'wwwdev.ebi.ac.uk/chembl/g/#browse/<BROWSER_NAME>/full_state/<GENERATED_STATE>'
,
'entity_from'
:
'CHEMBL_DRUG_WARNINGS'
,
'entity_to'
:
'CHEMBL_ACTIVITIES'
,
'es_query'
:
json
.
dumps
(
dataset_query
),
'selection_description'
:
json
.
dumps
(
selection_description
)
}
url
=
f
'
{
server_base_url
}
/entities_join/get_link_to_related_items'
print
(
'doing post: '
,
url
)
request
=
requests
.
post
(
url
,
data
=
join_params
)
print
(
'post done!'
)
status_code
=
request
.
status_code
print
(
f
'status_code:
{
status_code
}
'
)
response_text
=
request
.
text
utils
.
print_es_response
(
response_text
)
assert
status_code
==
200
,
'The request failed!'
functional_tests/specific_tests/fun_test_url_shortening.py
View file @
b438b304
...
...
@@ -50,7 +50,7 @@ def run_test(server_base_url, delayed_jobs_server_base_path):
utils
.
print_es_response
(
response_text
)
assert
status_code
==
200
,
'The request failed!'
time
.
sleep
(
5
)
# make it sleep to make sure the url is saved in ES
time
.
sleep
(
5
)
# make it sleep to make sure the url is saved in ES
response_json
=
request
.
json
()
long_url_got
=
response_json
.
get
(
'long_url'
)
...
...
functional_tests/specific_tests/utils.py
View file @
b438b304
"""
Module with utils functions for the tests
"""
import
json
import
time
import
requests
...
...
@@ -167,3 +168,12 @@ def wait_until_job_finished(delayed_jobs_server_base_path, job_id):
print
(
'job_status: '
,
job_status
)
print
(
'job_progress: '
,
job_progress
)
time
.
sleep
(
1
)
def
load_json_data
(
file_path
):
"""
:param file_path: path of the json file to load
:return: a dict with the data loaded
"""
with
open
(
file_path
)
as
the_file
:
return
json
.
loads
(
the_file
.
read
())
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment