Commit b5ec8769 authored by carlosribas's avatar carlosribas
Browse files

Merge remote-tracking branch 'remotes/origin/python3-version' into merge-master-python3

# Conflicts:
#	README.md
#	docker-compose.yml
#	jenkins/wp-kubernetes.jenkinsfile
#	rnacentral/apiv1/views.py
#	rnacentral/export/views.py
#	rnacentral/portal/config/expert_databases.py
#	rnacentral/portal/templates/portal/about.html
#	rnacentral/portal/templates/portal/header.html
#	rnacentral/portal/templates/portal/homepage.html
#	rnacentral/requirements.txt
#	rnacentral/rnacentral/settings.py
#	rnacentral/sequence_search/views.py
parents 14b22d96 32469434
S3_HOST=https://example.com
S3_KEY=RNAcentral_key
S3_SECRET=RNAcentral_secret
EBI_SEARCH_ENDPOINT=http://example.com # if not specified, www will be used
SECRET_KEY=super_secret_key # if not specified, it uses get_random_secret_key
DJANGO_DEBUG=True # if not specified, it uses DJANGO_DEBUG=False
LOCAL_DEVELOPMENT=True # use this variable if you need the django-debug-toolbar, coverage and mock packages
\ No newline at end of file
# This workflow will run the unit tests and create a new image on the Docker Hub.
# At least for now, the image will be created regardless of whether the tests are run successfully or not.
#
# References:
# - https://docs.github.com/en/actions/guides/building-and-testing-python
# - https://github.com/marketplace/actions/slack-notify
name: Building Docker containers
# Triggers the workflow on push or pull request events
on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies and rename local_settings file
run: |
python -m pip install --upgrade pip
pip install -r rnacentral/requirements.txt
pip install -r rnacentral/requirements_dev.txt
mv rnacentral/rnacentral/unit_test_local_settings.py rnacentral/rnacentral/local_settings.py
- name: Run tests
run: |
python rnacentral/manage.py test apiv1
python rnacentral/manage.py test export
python rnacentral/manage.py test portal
python rnacentral/manage.py test sequence_search
initial-notification:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Initial notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_MESSAGE: 'Creating a new image in the Docker Hub'
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
MSG_MINIMAL: true
create-docker-tag:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: docker login
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
- name: Get branch name
if: github.event_name != 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
- name: docker build
if: env.BRANCH_NAME != ''
run: docker build -f Dockerfile -t rnacentral/rnacentral-webcode:${{ env.BRANCH_NAME }} .
- name: docker push
if: env.BRANCH_NAME != ''
run: docker push rnacentral/rnacentral-webcode:${{ env.BRANCH_NAME }}
final-notification:
runs-on: ubuntu-latest
needs: create-docker-tag
steps:
- uses: actions/checkout@v2
- name: Final notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_MESSAGE: 'New image created in Docker Hub'
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
MSG_MINIMAL: true
\ No newline at end of file
......@@ -124,3 +124,6 @@ rnacentral/export/results
# environment variables
.env
# kubernetes
kubernetes/secrets
\ No newline at end of file
language: python
python:
- "2.7"
# install dependencies
install:
- "pip install requests selenium"
# run tests
script:
- travis_wait python rnacentral/portal/tests/selenium_tests.py --base_url=https://rnacentral.org --driver=phantomjs
# define branches
branches:
only:
- master
#------------------------------------------------------------------------------
#
# This Dockerfile is meant for containerized deployment with Kubernetes.
#
#-------------------------------------------------------------------------------
FROM python:3.8-slim
RUN apt-get update && apt-get install -y \
g++ \
build-essential \
curl \
tar \
git \
vim \
supervisor && \
rm -rf /var/lib/apt/lists/*
ENV RNACENTRAL_LOCAL=/srv/rnacentral/local
ENV SUPERVISOR_CONF_DIR=/srv/rnacentral/supervisor
ARG LOCAL_DEVELOPMENT
# Create folders
RUN \
mkdir -p $RNACENTRAL_LOCAL && \
mkdir -p $SUPERVISOR_CONF_DIR && \
mkdir /srv/rnacentral/log && \
mkdir /srv/rnacentral/static
# Install Infernal and node.js
RUN \
cd $RNACENTRAL_LOCAL && \
curl -OL http://eddylab.org/infernal/infernal-1.1.1.tar.gz && \
tar -xvzf infernal-1.1.1.tar.gz && \
cd infernal-1.1.1 && \
./configure --prefix=$RNACENTRAL_LOCAL/infernal-1.1.1 && \
make && \
make install && \
cd easel && \
make install && \
cd $RNACENTRAL_LOCAL && \
rm infernal-1.1.1.tar.gz && \
curl -sL https://deb.nodesource.com/setup_lts.x | bash - && \
apt-get install -y nodejs
# Create the rnacentral user
RUN useradd -m -d /srv/rnacentral -s /bin/bash rnacentral
# Set work directory
ENV RNACENTRAL_HOME=/srv/rnacentral/rnacentral-webcode
RUN mkdir -p $RNACENTRAL_HOME
WORKDIR $RNACENTRAL_HOME
# Copy requirements
COPY rnacentral/requirements.txt .
# Install requirements
RUN pip3 install -r requirements.txt
# Install NPM dependencies
ADD rnacentral/portal/static/package.json rnacentral/portal/static/
RUN cd rnacentral/portal/static && npm install --only=production
# Install R2DT-Web - used by test.rnacentral.org
RUN \
cd rnacentral/portal/static && \
git clone https://github.com/RNAcentral/r2dt-web.git && \
cd r2dt-web && \
git checkout dev
# Install rnacentral-sequence-search-embed - used by test.rnacentral.org
RUN \
cd rnacentral/portal/static && \
git clone https://github.com/RNAcentral/rnacentral-sequence-search-embed.git && \
cd rnacentral-sequence-search-embed && \
git checkout dev
# Copy and chown all the files to the rnacentral user
COPY rnacentral $RNACENTRAL_HOME/rnacentral
RUN chown -R rnacentral:rnacentral /srv
# Install and configure packages for local development if needed
RUN \
LOCAL_DEV="${LOCAL_DEVELOPMENT:-False}" && \
if [ "$LOCAL_DEV" = "True" ] ; then \
pip3 install -r rnacentral/requirements_dev.txt ; \
sed -i "13 a import debug_toolbar" "${RNACENTRAL_HOME}"/rnacentral/rnacentral/urls.py ; \
sed -i "31 a \ \ \ \ url(r'^__debug__/', include(debug_toolbar.urls))," "${RNACENTRAL_HOME}"/rnacentral/rnacentral/urls.py ; \
sed -i "129 a \ \ \ \ 'debug_toolbar.middleware.DebugToolbarMiddleware'," "${RNACENTRAL_HOME}"/rnacentral/rnacentral/settings.py ; \
sed -i "188 a \ \ \ \ 'debug_toolbar'," "${RNACENTRAL_HOME}"/rnacentral/rnacentral/settings.py ; \
fi
# Set user
USER rnacentral
# Run entrypoint
COPY ./entrypoint.sh $RNACENTRAL_HOME
ENTRYPOINT ["/srv/rnacentral/rnacentral-webcode/entrypoint.sh"]
# Supervisor
CMD [ "/bin/sh", "-c", "/usr/bin/supervisord -c ${SUPERVISOR_CONF_DIR}/supervisord.conf" ]
#------------------------------------------------------------------------------
# We have 2 ways to deploy RNAcentral in production:
# bare-metal managed (no root permissions) CentOS 6 VMs and Kubernetes.
#
# This dockerfile is meant to mimic the environment of
# bare-metal CentOS VMs in order to reate a reproducible
# installation of the RNAcentral website.
#
# All local dependencies are installed manually to mirror the
# bare-metal production setup on EBI VMs, where Docker or yum
# are not available.
#
#-------------------------------------------------------------------------------
FROM centos:6.10
RUN yum install -y \
curl \
gcc \
git \
httpd \
httpd-devel \
libaio \
"nc.x86_64" \
openssl \
openssl-devel \
tar \
unzip \
wget \
zlib-devel \
gcc-c++ \
make
RUN mkdir /rnacentral
RUN mkdir /rnacentral/local
RUN mkdir /rnacentral/static
ENV LOC /rnacentral/local
# Install Python
# NOTE: Python-2.7.11 and python-2.7.11 are DIFFERENT folders, the former contains the sources, the later - binaries
RUN \
cd $LOC && \
curl -OL http://www.python.org/ftp/python/2.7.11/Python-2.7.11.tgz && \
tar -zxvf Python-2.7.11.tgz && \
cd Python-2.7.11 && \
PREFIX=$LOC/python-2.7.11/ && \
export LD_RUN_PATH=$PREFIX/lib && \
./configure --prefix=$PREFIX --enable-shared && \
make && \
make install && \
cd $LOC && \
rm -Rf Python-2.7.11 && \
rm Python-2.7.11.tgz
# Install virtualenv
RUN \
cd $LOC && \
curl -OL https://pypi.python.org/packages/source/v/virtualenv/virtualenv-15.0.0.tar.gz && \
tar -zxvf virtualenv-15.0.0.tar.gz && \
cd virtualenv-15.0.0 && \
$LOC/python-2.7.11/bin/python setup.py install && \
cd $LOC && \
rm -Rf virtualenv-15.0.0.tar.gz && \
rm -Rf virtualenv-15.0.0
# Create RNAcentral virtual environment
RUN \
cd $LOC && \
mkdir virtualenvs && \
cd virtualenvs && \
$LOC/python-2.7.11/bin/virtualenv RNAcentral --python=$LOC/python-2.7.11/bin/python
# Install Redis (the stable version of Redis requires GCC 4.9 or above, not available for centos 6)
RUN \
cd $LOC && \
curl -OL http://download.redis.io/releases/redis-5.0.9.tar.gz && \
tar -xvzf redis-5.0.9.tar.gz && \
cd redis-5.0.9 && \
make && \
cd $LOC && \
rm redis-5.0.9.tar.gz && \
mv redis-5.0.9 redis
# Install libevent (memcached requirement)
RUN \
cd $LOC && \
curl --tlsv1 -OL https://github.com/downloads/libevent/libevent/libevent-2.0.21-stable.tar.gz && \
tar -zxvf libevent-2.0.21-stable.tar.gz && \
cd libevent-2.0.21-stable && \
./configure --prefix=$LOC/libevent && \
make && \
make install && \
cd $LOC && \
rm -Rf libevent-2.0.21-stable && \
rm libevent-2.0.21-stable.tar.gz
# Install memcached (requires libevent)
RUN \
cd $LOC && \
curl -OL http://www.memcached.org/files/memcached-1.4.17.tar.gz && \
tar -zxvf memcached-1.4.17.tar.gz && \
cd memcached-1.4.17 && \
./configure --prefix=$LOC/memcached --with-libevent=$LOC/libevent/ && \
make && \
make install && \
cd $LOC && \
rm -Rf memcached-1.4.17 && \
rm memcached-1.4.17.tar.gz
# Create a user for memcached
RUN adduser -g root xfm_adm
# Install Infernal
RUN \
cd $LOC && \
curl -OL http://eddylab.org/infernal/infernal-1.1.1.tar.gz && \
tar -xvzf infernal-1.1.1.tar.gz && \
cd infernal-1.1.1 && \
./configure --prefix=$LOC/infernal-1.1.1 && \
make && \
make install && \
cd easel && \
make install && \
cd $LOC && \
rm infernal-1.1.1.tar.gz
# Install mod_wsgi
RUN \
cd $LOC && \
mkdir httpd && \
mkdir httpd/modules && \
curl --tlsv1 -L -o mod_wsgi-3.4.tar.gz https://github.com/GrahamDumpleton/mod_wsgi/archive/3.4.tar.gz && \
tar -zxvf mod_wsgi-3.4.tar.gz && \
cd mod_wsgi-3.4 && \
export LD_RUN_PATH=$LOC/python-2.7.11/lib && \
export LD_LIBRARY_PATH=$LOC/python-2.7.11/lib/:$LD_LIBRARY_PATH && \
./configure --with-python=$LOC/python-2.7.11/bin/python && \
make && \
mv .libs/mod_wsgi.so $LOC/httpd/modules && \
cd $LOC && \
rm -Rf mod_wsgi-3.4 && \
rm mod_wsgi-3.4.tar.gz
# Install UCSC utilities
RUN \
cd $LOC && \
mkdir ucsc && \
cd ucsc && \
wget http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/fetchChromSizes && \
wget http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/bedToBigBed && \
chmod o+x fetchChromSizes && \
chmod o+x bedToBigBed
# Define container environment variables
ENV RNACENTRAL_HOME /rnacentral/rnacentral-webcode
ENV RNACENTRAL_LOCAL /rnacentral/local
# Install Django requirements
ADD rnacentral/requirements.txt $RNACENTRAL_HOME/rnacentral/
RUN \
source $LOC/virtualenvs/RNAcentral/bin/activate && \
pip install -r $RNACENTRAL_HOME/rnacentral/requirements.txt
# Install Node.js
RUN curl -sL https://rpm.nodesource.com/setup_6.x | bash -
RUN yum install -y nodejs
# Install NPM dependencies
ADD rnacentral/portal/static/package.json $RNACENTRAL_HOME/rnacentral/portal/static/
RUN cd $RNACENTRAL_HOME/rnacentral/portal/static && npm install --only=production
# Expose a container port where the website is served
EXPOSE 8000
# Start up the app
ENTRYPOINT \
source $LOC/virtualenvs/RNAcentral/bin/activate && \
supervisord -c $RNACENTRAL_HOME/supervisor/supervisor.conf && \
python $RNACENTRAL_HOME/rnacentral/manage.py runserver 0.0.0.0:8000
#------------------------------------------------------------------------------
#
# This Dockerfile is meant for containerized deployment with Kubernetes.
# This Dockerfile is meant for local development.
#
#-------------------------------------------------------------------------------
FROM debian:buster-slim
FROM python:3.8-slim
RUN apt-get update && apt-get install -y \
g++ \
......@@ -12,19 +12,22 @@ RUN apt-get update && apt-get install -y \
curl \
tar \
git \
python2.7 \
libpython2.7-dev \
python-pip \
vim \
supervisor && \
useradd -m -d /srv/rnacentral -s /bin/bash rnacentral
rm -rf /var/lib/apt/lists/*
ENV RNACENTRAL_HOME=/srv/rnacentral
ENV RNACENTRAL_LOCAL=$RNACENTRAL_HOME/local
ENV SUPERVISOR_CONF_DIR=${SUPERVISOR_CONF_DIR:-"/srv/rnacentral/supervisor"}
ENV RNACENTRAL_LOCAL=/srv/rnacentral/local
ENV SUPERVISOR_CONF_DIR=/srv/rnacentral/supervisor
# Create folders. Install Infernal and node.js
# Create folders
RUN \
mkdir -p $RNACENTRAL_LOCAL && \
mkdir -p $SUPERVISOR_CONF_DIR && \
mkdir /srv/rnacentral/log && \
mkdir /srv/rnacentral/static
# Install Infernal and node.js
RUN \
mkdir -p $RNACENTRAL_HOME/local && \
cd $RNACENTRAL_LOCAL && \
curl -OL http://eddylab.org/infernal/infernal-1.1.1.tar.gz && \
tar -xvzf infernal-1.1.1.tar.gz && \
......@@ -36,23 +39,25 @@ RUN \
make install && \
cd $RNACENTRAL_LOCAL && \
rm infernal-1.1.1.tar.gz && \
curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
curl -sL https://deb.nodesource.com/setup_lts.x | bash - && \
apt-get install -y nodejs
USER rnacentral
# Set work directory
ENV RNACENTRAL_HOME=/srv/rnacentral/rnacentral-webcode
RUN mkdir -p $RNACENTRAL_HOME
WORKDIR $RNACENTRAL_HOME
# Download RNAcentral, install requirements and node.js dependencies
RUN \
cd $RNACENTRAL_HOME && \
git clone https://github.com/RNAcentral/rnacentral-webcode.git && \
pip install -r $RNACENTRAL_HOME/rnacentral-webcode/rnacentral/requirements.txt && \
pip install gunicorn && \
cd $RNACENTRAL_HOME/rnacentral-webcode/rnacentral/portal/static && npm install --only=production
# Copy requirements
COPY rnacentral/requirements* .
COPY ./entrypoint.sh /entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
# Install requirements
RUN pip3 install -r requirements.txt && pip3 install -r requirements_dev.txt
# Expose a container port where the website is served
EXPOSE 8000
# Install NPM dependencies
COPY rnacentral/portal/static/package.json .
RUN npm install --only=production
CMD [ "/bin/sh", "-c", "/usr/bin/supervisord -c ${SUPERVISOR_CONF_DIR}/supervisord.conf" ]
# Run entrypoint
COPY ./entrypoint-local.sh /
RUN chmod 744 /entrypoint-local.sh
ENTRYPOINT ["/entrypoint-local.sh"]
#------------------------------------------------------------------------------
# We have 2 ways to deploy RNAcentral in production:
# bare-metal managed (no root permissions) CentOS 6 VMs and Kubernetes.
#
# This Dockerfile is meant for containerized deployment with Kubernetes.
#
#-------------------------------------------------------------------------------
FROM debian:latest
RUN apt-get update && apt-get install -y \
g++ \
build-essential \
curl \
wget \
tar \
git \
python2.7 \
libpython2.7-dev \
python-pip \
redis-server \
memcached
ADD . /srv/rnacentral
RUN mkdir /srv/rnacentral/local
RUN mkdir /srv/rnacentral/static
ENV RNACENTRAL_LOCAL /srv/rnacentral/local
ENV RNACENTRAL_HOME /srv/rnacentral
# Install Infernal
RUN \
cd $RNACENTRAL_LOCAL && \
curl -OL http://eddylab.org/infernal/infernal-1.1.1.tar.gz && \
tar -xvzf infernal-1.1.1.tar.gz && \
cd infernal-1.1.1 && \
./configure --prefix=$RNACENTRAL_LOCAL/infernal-1.1.1 && \
make && \
make install && \
cd easel && \
make install && \
cd $RNACENTRAL_LOCAL && \
rm infernal-1.1.1.tar.gz
# Install Django requirements
RUN pip install -r /srv/rnacentral/rnacentral/requirements.txt
# Install node.js
RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
RUN apt-get install -y nodejs
# Install node.js dependencies
ADD rnacentral/portal/static/package.json $RNACENTRAL_HOME/rnacentral/portal/static/
RUN cd $RNACENTRAL_HOME/rnacentral/portal/static && npm install --only=production
# Expose a container port where the website is served
EXPOSE 8000
# Start up the app
CMD python /srv/rnacentral/rnacentral/manage.py runserver 0.0.0.0:8000
#------------------------------------------------------------------------------
#
# This Dockerfile is for installing rq-dashboard and monitoring RQ queues.
#
#-------------------------------------------------------------------------------
FROM python:3.8-slim
RUN pip install rq-dashboard
RUN pip install click==7.1.2 # rq-dashboard==0.6.3 is incompatible with click==8.0.1
......@@ -19,14 +19,15 @@ The development of RNAcentral is coordinated by the
git clone --recursive https://github.com/RNAcentral/rnacentral-webcode.git
```
2. Edit database connection parameters in `rnacentral/local_settings.py`. Use the [public postgres database](https://rnacentral.org/help/public-database) when developing outside the EBI network.
2. Using the `.env-example` file as a template, create the `.env` file and change the environment variables as desired.
The [public postgres database](https://rnacentral.org/help/public-database) is configured by default.
If you want to use another database, add the following environment variables: `DB_HOST`, `DB_NAME`, `DB_USER`,
`DB_PASSWORD`, and `DB_PORT`.
3. Run the app using [Docker](https://www.docker.com):
```
export RNACENTRAL_HOME=/path/to/rnacentral/code
cd $RNACENTRAL_HOME
docker-compose up --build
docker-compose -f docker-compose.local.yml up --build
```
## Feedback
......
version: '3'
services:
# RNAcentral website
rnacentral:
build:
context: .
dockerfile: Dockerfile-local
environment:
- DB_HOST=${DB_HOST}
- DB_NAME=${DB_NAME}
- DB_USER=${DB_USER}
- DB_PASSWORD=${DB_PASSWORD}
- DB_PORT=${DB_PORT}
- SECRET_KEY=${SECRET_KEY}
- DJANGO_DEBUG=${DJANGO_DEBUG}
- S3_HOST=${S3_HOST}
- S3_KEY=${S3_KEY}
- S3_SECRET=${S3_SECRET}
- EBI_SEARCH_ENDPOINT=${EBI_SEARCH_ENDPOINT}
command: python rnacentral/manage.py runserver 0.0.0.0:8000
ports:
- 8000:8000
volumes:
- ./openssl/openssl.cnf:/etc/ssl/openssl.cnf
- ./rnacentral:/srv/rnacentral/rnacentral-webcode/rnacentral
- .env:/srv/rnacentral/rnacentral-webcode/.env
depends_on:
- redis
networks:
- redis-network
- memcached-network
# rqworker - used to export search results
rqworker:
build:
context: .
dockerfile: Dockerfile-local
environment:
- DB_HOST=${DB_HOST}
- DB_NAME=${DB_NAME}
- DB_USER=${DB_USER}
- DB_PASSWORD=${DB_PASSWORD}
- DB_PORT=${DB_PORT}
- SECRET_KEY=${SECRET_KEY}
- DJANGO_DEBUG=${DJANGO_DEBUG}
- S3_HOST=${S3_HOST}
- S3_KEY=${S3_KEY}
- S3_SECRET=${S3_SECRET}
- EBI_SEARCH_ENDPOINT=${EBI_SEARCH_ENDPOINT}
command: python rnacentral/manage.py rqworker
volumes:
- ./rnacentral:/srv/rnacentral/rnacentral-webcode/rnacentral
- .env:/srv/rnacentral/rnacentral-webcode/.env
depends_on:
- redis
networks:
- redis-network
# Redis server
redis:
image: redis:6.0.8-alpine
ports:
- 8051:8051