Merge branch 'production_ready' into 'master'

Production ready

See merge request pdbe-kb/services/pdbe-graph-api!382

.gitlab-ci.yml

 image: python:3.6-slim
 
 before_script:
-  - apt-get update && apt-get install -y apache2 apache2-dev libapache2-mod-wsgi-py3 build-essential git
-  - pip install -r requirements.txt
-  - easy_install bottle_memcache
+  - apt-get update && apt-get install -y build-essential git
+  - pip install -r requirements.dev.txt
+  # - easy_install bottle_memcache
   
   # make sure Neo4J connection details are provided as protected variables in Settings -> CI/CD -> Variables
   # this exports all variables so that connection details are available in environment

app/app.py

@@ -12,10 +12,6 @@ import pepvep
 import simplejson
 
 app = bottle.Bottle()
-mc_plugin = util_common.create_memcache_plugin()
-
-# Installs the memcache plugin
-app.install(mc_plugin)
 app.install(request_handler)
 
 mounts = {
@@ -149,7 +145,7 @@ reSlashOrNot = '<:re:/{0,1}>'
 """
 
 @app.route('/version')
-def api_version(mc):
+def api_version():
     import version
     return str(version.__version__)
 

app/compound.py

@@ -27,10 +27,6 @@ BONDLENGTH_QUANTIZE_DECIMAL = Decimal("0.001")
 
 
 app = bottle.Bottle()
-mc_plugin = util_common.create_memcache_plugin()
-
-# Installs the memcache plugin
-app.install(mc_plugin)
 
 # Installs the request handler plugin
 app.install(request_handler)
@@ -286,7 +282,7 @@ def get_compound_in_pdb(chem_comp_id):
 @apiExample {json=./examples/success/compound_co_factors.json} apiSuccessExample Example success response JSON
 """
 @app.get('/cofactors'+reSlashOrNot)
-def get_compound_co_factors_api(mc):
+def get_compound_co_factors_api():
 
     response, response_status = get_compound_co_factors()
 

app/config.py

@@ -29,16 +29,16 @@ for h, l in parser.items(level, raw=True):
     exec(h.upper() + "=" + l)
 
 # memcache configuration
-fmt = "{host}:{port}"
-port = "11211"
-MEMCACHE_SERVERS = [fmt.format(host=socket.getfqdn(), port=port)]
+# fmt = "{host}:{port}"
+# port = "11211"
+# MEMCACHE_SERVERS = [fmt.format(host=socket.getfqdn(), port=port)]
 
 print("=" * 80)
 
 print("Executing at %s level" % level.upper())
 
-print("MEMCACHE_DISTRIBUTED: %s" % MEMCACHE_DISTRIBUTED)
-print("MEMCACHE_SERVERS: %s" % MEMCACHE_SERVERS)
+# print("MEMCACHE_DISTRIBUTED: %s" % MEMCACHE_DISTRIBUTED)
+# print("MEMCACHE_SERVERS: %s" % MEMCACHE_SERVERS)
 
 print("NEO4J_DEST_URL: %s" % NEO4J_DEST_URL)
 print("OUTPUT_JSON_PATH: %s" % OUTPUT_JSON_PATH)

app/pdb.py

@@ -31,10 +31,6 @@ BONDLENGTH_QUANTIZE_DECIMAL = Decimal("0.001")
 list_re = r'[\s\']'
 
 app = bottle.Bottle()
-mc_plugin = util_common.create_memcache_plugin()
-
-# Installs the memcache plugin
-app.install(mc_plugin)
 
 # Installs the request handler plugin
 app.install(request_handler)

app/pdbe_pages.py

@@ -19,10 +19,6 @@ from uniprot import get_uniprot_variation_api
 print('[PDBE PAGES] Starting')
 
 app = bottle.Bottle()
-mc_plugin = util_common.create_memcache_plugin()
-
-# Installs the memcache plugin
-app.install(mc_plugin)
 
 # Installs the request handler plugin
 app.install(request_handler)
@@ -870,7 +866,7 @@ def get_annotations(pdb_id, entity_id):
 
     query = """
     MATCH (entry:Entry {ID:$pdb_id})-[:HAS_ENTITY]->(entity:Entity {ID:$entity_id})-[:HAS_PDB_RESIDUE]->(pdb_res:PDBResidue)<-[res_rel:FUNPDBE_ANNOTATION_FOR {CHAIN_LABEL:entity.BEST_CHAIN_ID}]-(fun_group:FunPDBeResidueGroup)-[:FUNPDBE_RESIDUE_GROUP_OF]->(fun_entry:FunPDBeEntry)
-    WHERE NOT fun_entry.DATA_RESOURCE IN ["FoldX", "3Dcomplex"]
+    WHERE NOT fun_entry.DATA_RESOURCE IN ["FoldX", "3Dcomplex","MetalPDB","ChannelsDB","POPScomp_PDBML","Missense3D"]
     WITH DISTINCT fun_entry.DATA_RESOURCE AS data_resource, fun_group.ORDINAL_ID AS ordinal_id, fun_group.LABEL AS group_label, fun_entry.RESOURCE_ENTRY_URL AS resource_url, toInteger(pdb_res.ID) AS pdb_res_id, pdb_res.CHEM_COMP_ID AS pdb_res_code, res_rel.CONFIDENCE_CLASSIFICATION AS conf_level, res_rel.CONFIDENCE_SCORE AS conf_score, res_rel.RAW_SCORE AS raw_score ORDER BY toInteger(pdb_res_id)
     RETURN DISTINCT data_resource, ordinal_id, group_label, resource_url, pdb_res_id AS pdb_residue, pdb_res_code AS pdb_residue_code, conf_level, conf_score, raw_score ORDER BY pdb_res_id
     """

app/pepvep.py

@@ -21,10 +21,6 @@ from uniprot import get_uniprot_sequence
 print('[PEPVEP] Starting')
 
 app = bottle.Bottle()
-mc_plugin = util_common.create_memcache_plugin()
-
-# Installs the memcache plugin
-app.install(mc_plugin)
 
 # Installs the request handler plugin
 app.install(request_handler)

app/plugins.py

@@ -78,17 +78,17 @@ def request_handler(fn):
                 if kwargs[key].count(',') > 999:
                     bottle.response.status = 400
                     return simplejson.dumps({}, indent=4)
-        else:
+        # else:
             
             # Check if it's a string and starts with '
-            if isinstance(kwargs[key], str) and len(kwargs[key]) and kwargs[key][0] != "'":
-                kwargs[key] = "'%s'" % kwargs[key]
-
+            # if isinstance(kwargs[key], str) and len(kwargs[key]) and kwargs[key][0] != "'":
+            #     kwargs[key] = "'%s'" % kwargs[key]
+        
             # If it matches the re convert to lowercase
-            if key != "mc" and re_list_PDB.match(kwargs[key]):
-                kwargs[key] = kwargs[key].lower()
+        #     if key != "mc" and re_list_PDB.match(kwargs[key]):
+        #         kwargs[key] = kwargs[key].lower()
 
-        mc = None
+        # mc = None
         output = None
 
         # Check if call needs to check for static files
@@ -109,33 +109,33 @@ def request_handler(fn):
                 pass
 
         # If the call requests caching
-        if bottle.request.method == 'GET' and kwargs.get("mc") is not None:
-            mc = kwargs["mc"]
+        # if bottle.request.method == 'GET' and kwargs.get("mc") is not None:
+        #     mc = kwargs["mc"]
             
-            cache_url = urlsplit(bottle.request.url).path
+        #     cache_url = urlsplit(bottle.request.url).path
 
-            if cache_url[-1] != "/":
-                cache_url += "/"
+        #     if cache_url[-1] != "/":
+        #         cache_url += "/"
 
-            try:
-                output = mc.get(cache_url)
-                bottle.response.status = mc.get(cache_url + "_status")
-            except Exception as e:
-                pass
+        #     try:
+        #         output = mc.get(cache_url)
+        #         bottle.response.status = mc.get(cache_url + "_status")
+        #     except Exception as e:
+        #         pass
 
 
         # Caching is not active
         if not output:
             output = fn(*args, **kwargs)
             
-            if bottle.request.method == 'GET' and mc:
-                try:
-                    # to overcome key-size < 250 issue, can't set it in ctor :-o
-                    mc.set(cache_url, output)
-                    # save the status code in the cache
-                    mc.set(cache_url + "_status", bottle.response.status)
-                except Exception as e:
-                    pass
+            # if bottle.request.method == 'GET' and mc:
+            #     try:
+            #         # to overcome key-size < 250 issue, can't set it in ctor :-o
+            #         mc.set(cache_url, output)
+            #         # save the status code in the cache
+            #         mc.set(cache_url + "_status", bottle.response.status)
+            #     except Exception as e:
+            #         pass
         
         if len(output) == 0:
             bottle.response.status = 404

app/residue.py

@@ -22,9 +22,6 @@ print('[RESIDUE] Starting')
 
 app = bottle.Bottle()
 
-mc_plugin = util_common.create_memcache_plugin()
-
-app.install(mc_plugin)
 app.install(request_handler)
 graph = neo4j_model.get_neo4j_instance()
 

app/sifts.py

@@ -21,9 +21,6 @@ print('[SIFTS] Starting')
 
 app = bottle.Bottle()
 
-mc_plugin = util_common.create_memcache_plugin()
-
-app.install(mc_plugin)
 app.install(request_handler)
 graph = neo4j_model.get_neo4j_instance()
 

app/uniprot.py

@@ -41,10 +41,6 @@ POOL_MAX_WORKERS = 10
 ALLOWED_OVERLAP_PERCENT_BEST_STRUCTURE = 30
 
 app = bottle.Bottle()
-mc_plugin = util_common.create_memcache_plugin()
-
-# Installs the memcache plugin
-app.install(mc_plugin)
 
 # Installs the request handler plugin
 app.install(request_handler)

app/util_common.py

 #!/usr/bin/env python
 
 import bottle
-from bottle.ext import memcache as bottle_memcache
 # import memcache as bottle_memcache
 import config
 from amino_acid_codes import amino_acid_codes, amino_acid_codes_one_to_three
@@ -11,11 +10,11 @@ PEPVEP_RECORD_LIMIT = 10
 PEPVEP_POSITION_LIMIT = 20
 
 
-def create_memcache_plugin():
-    return bottle_memcache.MemcachePlugin(
-        servers=config.MEMCACHE_SERVERS,
-        server_max_value_length=1024 * 1024 * 128
-    )
+# def create_memcache_plugin():
+#     return bottle_memcache.MemcachePlugin(
+#         servers=config.MEMCACHE_SERVERS,
+#         server_max_value_length=1024 * 1024 * 128
+#     )
 
 
 def split_nonconsecutive_residues(list_of_tuples):

app/validation.py

@@ -27,10 +27,6 @@ print('[VALIDATION] Starting')
 list_re = r'[\s\']'
 
 app = bottle.Bottle()
-mc_plugin = util_common.create_memcache_plugin()
-
-# Installs the memcache plugin
-app.install(mc_plugin)
 
 # Installs the request handler plugin
 app.install(request_handler)

requirements.dev.txt

+bottle==0.12.13
+py2neo==4.2.0
+simplejson==3.16.0
+urllib3==1.23
+pytest==4.4.0
+pytest-xdist==1.33.0
+tavern==0.22.1
+requests==2.22.0
+configparser==3.7.1
+deepdiff==3.3.0
+numpy==1.16.1
+boom==1.0
+neobolt==1.7.16
+neotime==1.7.4
+more-itertools==4.3.0
+git+https://gitlab.ebi.ac.uk/pdbe-kb/services/generic-to-protvista-adapter.git#egg=protvista_adapter

requirements.txt

-bottle
-py2neo
-simplejson
+bottle==0.12.13
+py2neo==4.2.0
+simplejson==3.16.0
 urllib3==1.23
-pytest
-pytest-xdist
-tavern
-requests
-configparser
-deepdiff
-numpy
-boom
-neobolt
-neotime
+configparser==3.7.1
+boom==1.0
+neobolt==1.7.16
+neotime==1.7.4
+more-itertools==4.3.0
 git+https://gitlab.ebi.ac.uk/pdbe-kb/services/generic-to-protvista-adapter.git#egg=protvista_adapter

setup.sh

@@ -17,16 +17,20 @@ echo "Setting up ${environment} environment"
 CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 echo "Creating Python environment"
-if [[ $environment == 'local' ]]; then
-    python3 -m venv $CURR_DIR/venv
-else
-    python3.6 -m venv $CURR_DIR/venv
-fi
+# if [[ $environment == 'local' ]]; then
+#     python3 -m venv $CURR_DIR/venv
+# else
+#     python3.6 -m venv $CURR_DIR/venv
+# fi
+
+python3 -m venv $CURR_DIR/venv
 
 if [[ $environment == 'dev' ]]; then
     git checkout development
 fi
 
+python3 -m venv $CURR_DIR/venv
+
 source $CURR_DIR/venv/bin/activate
 
 echo "Upgrade pip and setuptools"
@@ -40,8 +44,6 @@ fi
 echo "Install project specific Python modules"
 pip install -r $CURR_DIR/requirements.txt
 
-easy_install bottle_memcache
-
 if [[ $environment != 'local' ]]; then
     echo "Cloning pdbe-graph-api-config to get config"
     git clone git@gitlab.ebi.ac.uk:pdbe-kb/pdbe-graph-api-config.git