Unverified Commit 46b0c34b authored by Carlos Eduardo Ribas's avatar Carlos Eduardo Ribas Committed by GitHub
Browse files

Merge pull request #534 from RNAcentral/release-17

Release 17
parents 874ad580 2c85727b
......@@ -4,11 +4,12 @@
## About
RNAcentral is an open public resource that offers integrated access to a comprehensive and up-to-date set of ncRNA sequences. For more information, please visit https://rnacentral.org/about-us.
[RNAcentral](https://rnacentral.org) is an open public resource that offers integrated access to a comprehensive and up-to-date set of non-coding RNA sequences provided by a consortium of Expert Databases.
![RNAcentral Expert Databases](./rnacentral/portal/static/img/expert-databases.png)
The development of RNAcentral is coordinated by the
[European Bioinformatics Institute](http://www.ebi.ac.uk) and is funded by the
[BBSRC](http://www.bbsrc.ac.uk).
[European Bioinformatics Institute](https://www.ebi.ac.uk) and is funded by [Wellcome](https://wellcome.org).
## Installation
......@@ -28,58 +29,7 @@ The development of RNAcentral is coordinated by the
docker-compose up --build
```
**Docker Cheat Sheet**
```
# connect to a running container
docker ps
docker exec -it <container_id> bash
```
### API tests
1. Login to the running Docker container (see above).
1. Launch tests:
```sh
cd $RNACENTRAL_HOME
python rnacentral/apiv1/tests.py
```
## Feedback
Feel free to give feedback using [GitHub issues](https://github.com/RNAcentral/rnacentral-webcode/issues)
or get in touch using the [Contact form](https://rnacentral.org/contact) on our website.
## Technology overview
### Python
* [Django](https://www.djangoproject.com/)
* [Django REST Framework](http://www.django-rest-framework.org/)
* [Fabric](http://www.fabfile.org/)
* [Python RQ](http://python-rq.org/)
* see [requirements.txt](rnacentral/requirements.txt) for the full list
### Javascript
* [AngularJS](https://angularjs.org/)
* [Twitter Bootstrap](http://getbootstrap.com/)
* [D3](http://d3js.org/)
* [jQuery](https://jquery.com/)
* [Genoverse](http://genoverse.org)
* see [LICENSE](LICENSE) for more details
### CSS
* [Twitter Bootstrap](http://getbootstrap.com/)
* [Font Awesome](http://fontawesome.io/)
* [Animate.css](https://daneden.github.io/animate.css/)
* see [LICENSE](LICENSE) for more details
### Other
* [Memcached](http://memcached.org/)
* [Redis](http://redis.io/)
* [Supervisor](http://supervisord.org/)
* [Docker](https://www.docker.com)
Please get in touch if you have any feedback using [GitHub issues](https://github.com/RNAcentral/rnacentral-webcode/issues)
or using the [contact us form](https://rnacentral.org/contact).
......@@ -11,11 +11,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import boto3
import re
import requests
import zlib
from itertools import chain
from django.conf import settings
from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404
from django_filters import rest_framework as filters
......@@ -364,33 +366,29 @@ class SecondaryStructureSVGImage(generics.ListAPIView):
permission_classes = (AllowAny,)
def get(self, request, pk=None, format=None):
ftp = "http://ftp.ebi.ac.uk/pub/databases/RNAcentral/releases/15.0/.secondary-structure/secondary-structure/{}.svg.gz"
s3 = boto3.resource(
's3',
aws_access_key_id=settings.S3_SERVER['KEY'],
aws_secret_access_key=settings.S3_SERVER['SECRET'],
endpoint_url=settings.S3_SERVER['HOST']
)
upi = list(self.kwargs['pk'])
upi_path = "".join(upi[0:3]) + "/" \
+ "".join(upi[3:5]) + "/" \
+ "".join(upi[5:7]) + "/" \
+ "".join(upi[7:9]) + "/" \
+ "".join(upi[9:11]) + "/"
url = ftp.format(upi_path + "".join(upi))
s3_file = "prod/" + upi_path + self.kwargs['pk'] + ".svg.gz"
s3_obj = s3.Object(settings.S3_SERVER['BUCKET'], s3_file)
try:
response = requests.get(url)
response.raise_for_status()
svg_ftp = zlib.decompress(response.content, zlib.MAX_WBITS | 32)
except requests.exceptions.HTTPError as e:
svg_ftp = None
try:
svg_bd = SecondaryStructureWithLayout.objects.get(urs="".join(upi))
svg_bd = svg_bd.layout
except SecondaryStructureWithLayout.DoesNotExist:
svg_bd = None
if not svg_ftp and not svg_bd:
s3_svg = zlib.decompress(s3_obj.get()['Body'].read(), zlib.MAX_WBITS | 32)
except s3.meta.client.exceptions.NoSuchKey:
return Response(status=status.HTTP_404_NOT_FOUND)
return HttpResponse(
self.generate_thumbnail(svg_ftp if svg_ftp else svg_bd, "".join(upi)), content_type='image/svg+xml'
self.generate_thumbnail(s3_svg, "".join(upi)), content_type='image/svg+xml'
)
def generate_thumbnail(self, image, upi):
......
......@@ -91,7 +91,7 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': 'as of July 20th, 2020',
'version': 'as of December 20th, 2020',
},
{
'name': 'FlyBase',
......@@ -116,7 +116,7 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': 'FB2020_03',
'version': 'FB2020_06',
},
{
'name': 'Ensembl',
......@@ -147,7 +147,7 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': '100',
'version': '102',
},
{
'name': 'Ensembl Plants',
......@@ -172,7 +172,7 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': '47',
'version': '49',
},
{
'name': 'Ensembl Fungi',
......@@ -197,7 +197,7 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': '47',
'version': '49',
},
{
'name': 'Ensembl Metazoa',
......@@ -222,7 +222,7 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': '47',
'version': '49',
},
{
'name': 'Ensembl Protists',
......@@ -247,11 +247,11 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': '47',
'version': '49',
},
{
'name': 'GENCODE',
'label': 'gencode',
'name': 'Ensembl/GENCODE',
'label': 'ensembl_gencode',
'url': 'http://gencodegenes.org/',
'description': 'produces high quality reference gene annotation and experimental validation for human and mouse genomes',
'hint': 'GENCODE produces high quality reference gene annotation and experimental validation for human and mouse genomes',
......@@ -272,7 +272,7 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': 'human 25/mouse M12',
'version': 'human 37/mouse M26',
},
{
'name': 'Rfam',
......@@ -502,7 +502,7 @@ expert_dbs = [
],
'imported': True,
'status': 'updated',
'version': '201', # ftp://ftp.ncbi.nlm.nih.gov/refseq/release/RELEASE_NUMBER
'version': '204', # ftp://ftp.ncbi.nlm.nih.gov/refseq/release/RELEASE_NUMBER
},
{
'name': 'RDP',
......@@ -551,7 +551,7 @@ expert_dbs = [
},
],
'imported': True,
'status': 'new',
'status': '',
'version': '',
},
{
......@@ -726,15 +726,28 @@ expert_dbs = [
},
{
'name': 'piRBase',
'label': '',
'label': 'pirbase',
'url': 'http://www.regulatoryrna.org/database/piRNA/',
'description': 'a web resource assisting piRNA functional study',
'hint': 'piRBase is a web resource assisting piRNA functional study',
'description': 'a database of various piRNA associated data to support piRNA functional study ',
'hint': 'piRBase is a database of various piRNA associated data to support piRNA functional study',
'tags': ['automatic', 'curated', 'piRNA'],
'abbreviation': '',
'examples': '',
'references': [],
'imported': False,
'examples': [
{'upi': 'URS0000061ED0', 'taxid': 9606},
{'upi': 'URS0000298B36', 'taxid': 10090},
{'upi': 'URS00000FDFAF', 'taxid': 7227},
],
'references': [
{
'title': 'piRBase: a comprehensive database of piRNA sequences',
'authors': 'Wang J, Zhang P, Lu Y, Li Y, Zheng Y, Kan Y, Chen R, He S',
'journal': 'Nucleic Acids Res. 2019 Jan 8;47(D1):D175-D180',
'pubmed_id': '30371818',
},
],
'imported': True,
'status': 'New',
'version': '2.0 (only sequences matching existing RNAcentral accessions have been imported)'
},
{
'name': 'PLncDB',
......@@ -917,7 +930,7 @@ expert_dbs = [
},
],
'imported': True,
'status': 'new',
'status': '',
'version': '',
},
{
......@@ -1379,7 +1392,7 @@ expert_dbs = [
},
],
'imported': True,
'status': 'new',
'status': 'updated',
'version': '',
},
{
......
......@@ -216,7 +216,7 @@ class Accession(models.Model):
'MALACARDS': 'https://www.genecards.org/cgi-bin/carddisp.pl?gene={id}#diseases',
'GENECARDS': 'https://www.genecards.org/cgi-bin/carddisp.pl?gene={id}',
}
if self.database in ['GTRNADB', 'ZWD', 'SNODB', 'MIRGENEDB', '5SRRNADB', 'SILVA', 'SNORNADB', 'ZFIN']:
if self.database in ['GTRNADB', 'ZWD', 'SNODB', 'MIRGENEDB', '5SRRNADB', 'SILVA', 'SNORNADB', 'ZFIN', 'PIRBASE']:
try:
data = json.loads(self.note)
url = data['url'] if 'url' in data else ''
......
......@@ -11,6 +11,7 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
import boto3
import six
import operator as op
import itertools as it
......@@ -542,21 +543,26 @@ class Rna(CachingMixin, models.Model):
if not layout:
return {}
# added for release-16. Layout comes from the FTP
ftp = "http://ftp.ebi.ac.uk/pub/databases/RNAcentral/releases/15.0/.secondary-structure/secondary-structure/{}.svg.gz"
# Layout comes from S3
s3 = boto3.resource(
's3',
aws_access_key_id=settings.S3_SERVER['KEY'],
aws_secret_access_key=settings.S3_SERVER['SECRET'],
endpoint_url=settings.S3_SERVER['HOST']
)
upi = list(self.pk)
upi_path = "".join(upi[0:3]) + "/" \
+ "".join(upi[3:5]) + "/" \
+ "".join(upi[5:7]) + "/" \
+ "".join(upi[7:9]) + "/" \
+ "".join(upi[9:11]) + "/"
url = ftp.format(upi_path + "".join(upi))
s3_file = "prod/" + upi_path + self.pk + ".svg.gz"
s3_obj = s3.Object(settings.S3_SERVER['BUCKET'], s3_file)
try:
response = requests.get(url)
response.raise_for_status()
svg = zlib.decompress(response.content, zlib.MAX_WBITS | 32)
except requests.exceptions.HTTPError as e:
svg = zlib.decompress(s3_obj.get()['Body'].read(), zlib.MAX_WBITS | 32)
except s3.meta.client.exceptions.NoSuchKey:
svg = None
# model_name = layout.template.model_name
......@@ -575,7 +581,7 @@ class Rna(CachingMixin, models.Model):
'secondary_structure': layout.secondary_structure,
'source': layout.template.model_source,
'model_id': layout.template.model_name,
'layout': svg if svg else layout.layout,
'layout': svg,
'template_species': layout.template.taxid.name,
'template_lineage': layout.template.taxid.lineage,
}
......
......@@ -7,7 +7,7 @@ angular.module("expertDatabase").service('normalizeExpertDbName', ['routes', fun
*/
nameToImageUrl: function(name) {
if (name === "tmRNA Website") name = 'tmrna-website';
else name = name.toLowerCase().replace(' ', '_');
else name = name.toLowerCase().replace(' ', '_').replace('/', '_');
return routes.expertDbLogo({ expertDbName: name });
},
......
......@@ -91,6 +91,12 @@
<a class="gtrnadb-external-url" ng-href="{{ xref.accession.expert_db_url }}" target="_blank">{{ xref.accession.optional_id }}</a>
</span>
<!-- piRBase -->
<span ng-switch-when="piRBase">
piRBase:
<a class="pirbase-external-url" ng-href="{{ xref.accession.expert_db_url }}" target="_blank">{{ xref.accession.external_id }}</a>
</span>
<!-- miRBase -->
<span ng-switch-when="miRBase">
miRBase:
......
......@@ -606,7 +606,7 @@ About RNAcentral
},
"text": {
"headline": "NAR paper published",
"text": "The latest <a href='http://dx.doi.org/10.1093/nar/gky1034'>RNAcentral paper</a> has been published in Nucleic Acids Research."
"text": "New <a href='http://dx.doi.org/10.1093/nar/gky1034'>RNAcentral paper</a> has been published in Nucleic Acids Research."
}
},
{
......@@ -752,7 +752,49 @@ About RNAcentral
},
"text": {
"headline": "Release 16",
"text": "RNAcentral now includes ZFIN, CRW, and snoRNA Database, as well as SO annotations and millions of new 2D structures <br> <a href='{{ blog_url }}'>Release announcement</a>"
"text": "RNAcentral now includes ZFIN, CRW, and snoRNA Database, as well as SO annotations and millions of new 2D structures <br> <a href='https://blog.rnacentral.org/2020/09/rnacentral-release-16.html'>Release announcement</a>"
}
},
{
"media": {
"url": "{% static "img/timeline/nar-2021.png" %}",
"caption": "Paper in NAR",
"credit": "NAR"
},
"start_date": {
"month": "10",
"day": "27",
"year": "2020"
},
"end_date": {
"month": "10",
"day": "27",
"year": "2020"
},
"text": {
"headline": "NAR paper published",
"text": "New <a href='http://dx.doi.org/10.1093/nar/gkaa921'>RNAcentral paper</a> has been published in Nucleic Acids Research."
}
},
{
"media": {
"url": "{% static "img/timeline/release-v17.png" %}",
"caption": "Release 17 features",
"credit": "RNAcentral"
},
"start_date": {
"month": "03",
"day": "05",
"year": "2021"
},
"end_date": {
"month": "03",
"day": "05",
"year": "2021"
},
"text": {
"headline": "Release 17",
"text": "RNAcentral now includes piRBase as well as updated RNAse P diagrams <br> <a href='{{ blog_url }}'>Release announcement</a>"
}
},
]
......@@ -882,6 +924,12 @@ About RNAcentral
'expert_databases': 44,
'label': 'release 16',
},
{
'date': '04-03-21',
'sequence_count': 30000958,
'expert_databases': 45,
'label': 'release 17',
},
// select count(*) from rnc_rna_precomputed rrp where taxid is not null and is_active is true and is_fragment is false;
];
......
......@@ -37,7 +37,7 @@ limitations under the License.
<span class="icon-bar"></span>
</button>
<p class="navbar-text">
<span class="badge release-number">v16</span>
<span class="badge release-number">v17</span>
</p>
</div>
<div class="navbar-collapse collapse">
......
......@@ -178,7 +178,7 @@ RNAcentral: The non-coding RNA sequence database
<div class="panel-body">
<p class="margin-top-5px">
<a href="{{ context.blog_url }}" class="thumbnail">
<img class="hvr-grow" src="{% static 'img/timeline/release-v16.png' %}" alt="Release announcement">
<img class="hvr-grow" src="{% static 'img/timeline/release-v17.png' %}" alt="Release announcement">
</a>
</p>
<ul class="list-inline margin-top-5px" style="margin-left: 5px;">
......@@ -187,7 +187,7 @@ RNAcentral: The non-coding RNA sequence database
</ul>
<p class="margin-top-5px">
<span class="label label-success">New</span> Read new RNAcentral paper in <a href="https://currentprotocols.onlinelibrary.wiley.com/doi/full/10.1002/cpbi.104" target="_blank">Current Protocols in Bioinformatics</a>
Read RNAcentral paper in <a href="https://currentprotocols.onlinelibrary.wiley.com/doi/full/10.1002/cpbi.104" target="_blank">Current Protocols in Bioinformatics</a>
</p>
</div>
</div>
......
......@@ -24,6 +24,7 @@ sqlparse==0.1.19
versiontools==1.9.1
simple-crypt==4.1.7
colorhash==1.0.2
boto3==1.17.18
# code style and test coverage reporting
django-jenkins==0.110.0
......@@ -65,3 +66,6 @@ django-performance-testing==0.5.0
# json schema
jsonschema==2.6.0
# AWS SDK for Python
boto3==1.16.24
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment