Fetch SVG from S3

rnacentral/apiv1/views.py

@@ -11,11 +11,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+import boto3
 import re
 import requests
 import zlib
 from itertools import chain
 
+from django.conf import settings
 from django.http import Http404, HttpResponse
 from django.shortcuts import get_object_or_404
 from django_filters import rest_framework as filters
@@ -364,33 +366,29 @@ class SecondaryStructureSVGImage(generics.ListAPIView):
     permission_classes = (AllowAny,)
 
     def get(self, request, pk=None, format=None):
-        ftp = "http://ftp.ebi.ac.uk/pub/databases/RNAcentral/current_release/.secondary-structure/secondary-structure/{}.svg.gz"
+        s3 = boto3.resource(
+            's3',
+            aws_access_key_id=settings.S3_SERVER['KEY'],
+            aws_secret_access_key=settings.S3_SERVER['SECRET'],
+            endpoint_url=settings.S3_SERVER['HOST']
+        )
+
         upi = list(self.kwargs['pk'])
         upi_path = "".join(upi[0:3]) + "/" \
                    + "".join(upi[3:5]) + "/" \
                    + "".join(upi[5:7]) + "/" \
                    + "".join(upi[7:9]) + "/" \
                    + "".join(upi[9:11]) + "/"
-        url = ftp.format(upi_path + "".join(upi))
 
+        s3_file = "prod/" + upi_path + self.kwargs['pk'] + ".svg.gz"
+        s3_obj = s3.Object(settings.S3_SERVER['BUCKET'], s3_file)
         try:
-            response = requests.get(url)
-            response.raise_for_status()
-            svg_ftp = zlib.decompress(response.content, zlib.MAX_WBITS | 32)
-        except requests.exceptions.HTTPError as e:
-            svg_ftp = None
-
-        try:
-            svg_bd = SecondaryStructureWithLayout.objects.get(urs="".join(upi))
-            svg_bd = svg_bd.layout
-        except SecondaryStructureWithLayout.DoesNotExist:
-            svg_bd = None
-
-        if not svg_ftp and not svg_bd:
+            s3_svg = zlib.decompress(s3_obj.get()['Body'].read(), zlib.MAX_WBITS | 32)
+        except s3.meta.client.exceptions.NoSuchKey:
             return Response(status=status.HTTP_404_NOT_FOUND)
 
         return HttpResponse(
-            self.generate_thumbnail(svg_ftp if svg_ftp else svg_bd, "".join(upi)), content_type='image/svg+xml'
+            self.generate_thumbnail(s3_svg, "".join(upi)), content_type='image/svg+xml'
         )
 
     def generate_thumbnail(self, image, upi):

rnacentral/portal/models/rna.py

@@ -11,6 +11,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import boto3
 import six
 import operator as op
 import itertools as it
@@ -541,21 +542,26 @@ class Rna(CachingMixin, models.Model):
         if not layout:
             return {}
 
-        # added for release-16. Layout comes from the FTP
-        ftp = "http://ftp.ebi.ac.uk/pub/databases/RNAcentral/current_release/.secondary-structure/secondary-structure/{}.svg.gz"
+        # Layout comes from S3
+        s3 = boto3.resource(
+            's3',
+            aws_access_key_id=settings.S3_SERVER['KEY'],
+            aws_secret_access_key=settings.S3_SERVER['SECRET'],
+            endpoint_url=settings.S3_SERVER['HOST']
+        )
+
         upi = list(self.pk)
         upi_path = "".join(upi[0:3]) + "/" \
                    + "".join(upi[3:5]) + "/" \
                    + "".join(upi[5:7]) + "/" \
                    + "".join(upi[7:9]) + "/" \
                    + "".join(upi[9:11]) + "/"
-        url = ftp.format(upi_path + "".join(upi))
 
+        s3_file = "prod/" + upi_path + self.pk + ".svg.gz"
+        s3_obj = s3.Object(settings.S3_SERVER['BUCKET'], s3_file)
         try:
-            response = requests.get(url)
-            response.raise_for_status()
-            svg = zlib.decompress(response.content, zlib.MAX_WBITS | 32)
-        except requests.exceptions.HTTPError as e:
+            svg = zlib.decompress(s3_obj.get()['Body'].read(), zlib.MAX_WBITS | 32)
+        except s3.meta.client.exceptions.NoSuchKey:
             svg = None
 
         # model_name = layout.template.model_name
@@ -574,7 +580,7 @@ class Rna(CachingMixin, models.Model):
             'secondary_structure': layout.secondary_structure,
             'source': layout.template.model_source,
             'model_id': layout.template.model_name,
-            'layout': svg if svg else layout.layout,
+            'layout': svg,
             'template_species': layout.template.taxid.name,
             'template_lineage': layout.template.taxid.lineage,
         }

rnacentral/requirements.txt

@@ -65,4 +65,7 @@ django-cache-machine==1.1.0
 # django-performance-testing==0.5.0
 
 # simplified static file serving for Python web apps
-whitenoise==5.2.0
\ No newline at end of file
+whitenoise==5.2.0
+
+# AWS SDK for Python
+boto3==1.16.24
\ No newline at end of file