Skip to content

GitLab

Unverified Commit 494d6073 authored by Carlos Eduardo Ribas's avatar Carlos Eduardo Ribas Committed by GitHub
Browse files

Merge pull request #552 from RNAcentral/master-python3 

Update the master branch with the version used in Kubernetes
parents 8103aca9 7641721c
Hide whitespace changes
Inline Side-by-side
Showing with 723 additions and 713 deletions
.env-example 0 → 100644
View file @ 494d6073
S3_HOST=https://example.com
S3_KEY=RNAcentral_key
S3_SECRET=RNAcentral_secret
EBI_SEARCH_ENDPOINT=http://example.com # if not specified, www will be used
SECRET_KEY=super_secret_key # if not specified, it uses get_random_secret_key
DJANGO_DEBUG=True # if not specified, it uses DJANGO_DEBUG=False
LOCAL_DEVELOPMENT=True # use this variable if you need the django-debug-toolbar, coverage and mock packages
\ No newline at end of file
.github/workflows/main.yml 0 → 100644
View file @ 494d6073
# This workflow will run the unit tests and create a new image on the Docker Hub.
# At least for now, the image will be created regardless of whether the tests are run successfully or not.
#
# References:
# - https://docs.github.com/en/actions/guides/building-and-testing-python
# - https://github.com/marketplace/actions/slack-notify
name: Building Docker containers
# Triggers the workflow on push or pull request events
on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies and rename local_settings file
run: |
python -m pip install --upgrade pip
pip install -r rnacentral/requirements.txt
pip install -r rnacentral/requirements_dev.txt
mv rnacentral/rnacentral/unit_test_local_settings.py rnacentral/rnacentral/local_settings.py
- name: Run tests
run: |
python rnacentral/manage.py test apiv1
python rnacentral/manage.py test export
python rnacentral/manage.py test portal
python rnacentral/manage.py test sequence_search
initial-notification:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Initial notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_MESSAGE: 'Creating a new image in the Docker Hub'
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
MSG_MINIMAL: true
create-docker-tag:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: docker login
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
- name: Get branch name
if: github.event_name != 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
- name: docker build
if: env.BRANCH_NAME != ''
run: docker build -f Dockerfile -t rnacentral/rnacentral-webcode:${{ env.BRANCH_NAME }} .
- name: docker push
if: env.BRANCH_NAME != ''
run: docker push rnacentral/rnacentral-webcode:${{ env.BRANCH_NAME }}
final-notification:
runs-on: ubuntu-latest
needs: create-docker-tag
steps:
- uses: actions/checkout@v2
- name: Final notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_MESSAGE: 'New image created in Docker Hub'
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
MSG_MINIMAL: true
\ No newline at end of file
.gitignore
View file @ 494d6073
......@@ -124,3 +124,6 @@ rnacentral/export/results
# environment variables
.env
# kubernetes
kubernetes/secrets
\ No newline at end of file
.travis.yml deleted 100644 → 0
View file @ 8103aca9
language: python
python:
- "2.7"
# install dependencies
install:
- "pip install requests selenium"
# run tests
script:
- travis_wait python rnacentral/portal/tests/selenium_tests.py --base_url=https://rnacentral.org --driver=phantomjs
# define branches
branches:
only:
- master
Dockerfile 0 → 100644
View file @ 494d6073
#------------------------------------------------------------------------------
#
# This Dockerfile is meant for containerized deployment with Kubernetes.
#
#-------------------------------------------------------------------------------
FROM python:3.8-slim
RUN apt-get update && apt-get install -y \
g++ \
build-essential \
curl \
tar \
git \
vim \
supervisor && \
rm -rf /var/lib/apt/lists/*
ENV RNACENTRAL_LOCAL=/srv/rnacentral/local
ENV SUPERVISOR_CONF_DIR=/srv/rnacentral/supervisor
ARG LOCAL_DEVELOPMENT
# Create folders
RUN \
mkdir -p $RNACENTRAL_LOCAL && \
mkdir -p $SUPERVISOR_CONF_DIR && \
mkdir /srv/rnacentral/log && \
mkdir /srv/rnacentral/static
# Install Infernal and node.js
RUN \
cd $RNACENTRAL_LOCAL && \
curl -OL http://eddylab.org/infernal/infernal-1.1.1.tar.gz && \
tar -xvzf infernal-1.1.1.tar.gz && \
cd infernal-1.1.1 && \
./configure --prefix=$RNACENTRAL_LOCAL/infernal-1.1.1 && \
make && \
make install && \
cd easel && \
make install && \
cd $RNACENTRAL_LOCAL && \
rm infernal-1.1.1.tar.gz && \
curl -sL https://deb.nodesource.com/setup_lts.x | bash - && \
apt-get install -y nodejs
# Create the rnacentral user
RUN useradd -m -d /srv/rnacentral -s /bin/bash rnacentral
# Set work directory
ENV RNACENTRAL_HOME=/srv/rnacentral/rnacentral-webcode
RUN mkdir -p $RNACENTRAL_HOME
WORKDIR $RNACENTRAL_HOME
# Copy requirements
COPY rnacentral/requirements.txt .
# Install requirements
RUN pip3 install -r requirements.txt
# Install NPM dependencies
ADD rnacentral/portal/static/package.json rnacentral/portal/static/
RUN cd rnacentral/portal/static && npm install --only=production
# Install R2DT-Web - used by test.rnacentral.org
RUN \
cd rnacentral/portal/static && \
git clone https://github.com/RNAcentral/r2dt-web.git && \
cd r2dt-web && \
git checkout dev
# Install rnacentral-sequence-search-embed - used by test.rnacentral.org
RUN \
cd rnacentral/portal/static && \
git clone https://github.com/RNAcentral/rnacentral-sequence-search-embed.git && \
cd rnacentral-sequence-search-embed && \
git checkout dev
# Copy and chown all the files to the rnacentral user
COPY rnacentral $RNACENTRAL_HOME/rnacentral
RUN chown -R rnacentral:rnacentral /srv
# Install and configure packages for local development if needed
RUN \
LOCAL_DEV="${LOCAL_DEVELOPMENT:-False}" && \
if [ "$LOCAL_DEV" = "True" ] ; then \
pip3 install -r rnacentral/requirements_dev.txt ; \
sed -i "13 a import debug_toolbar" "${RNACENTRAL_HOME}"/rnacentral/rnacentral/urls.py ; \
sed -i "31 a \ \ \ \ url(r'^__debug__/', include(debug_toolbar.urls))," "${RNACENTRAL_HOME}"/rnacentral/rnacentral/urls.py ; \
sed -i "129 a \ \ \ \ 'debug_toolbar.middleware.DebugToolbarMiddleware'," "${RNACENTRAL_HOME}"/rnacentral/rnacentral/settings.py ; \
sed -i "188 a \ \ \ \ 'debug_toolbar'," "${RNACENTRAL_HOME}"/rnacentral/rnacentral/settings.py ; \
fi
# Set user
USER rnacentral
# Run entrypoint
COPY ./entrypoint.sh $RNACENTRAL_HOME
ENTRYPOINT ["/srv/rnacentral/rnacentral-webcode/entrypoint.sh"]
# Supervisor
CMD [ "/bin/sh", "-c", "/usr/bin/supervisord -c ${SUPERVISOR_CONF_DIR}/supervisord.conf" ]
Dockerfile-development-backup deleted 100644 → 0
View file @ 8103aca9
#------------------------------------------------------------------------------
# We have 2 ways to deploy RNAcentral in production:
# bare-metal managed (no root permissions) CentOS 6 VMs and Kubernetes.
#
# This dockerfile is meant to mimic the environment of
# bare-metal CentOS VMs in order to reate a reproducible
# installation of the RNAcentral website.
#
# All local dependencies are installed manually to mirror the
# bare-metal production setup on EBI VMs, where Docker or yum
# are not available.
#
#-------------------------------------------------------------------------------
FROM centos:6.10
RUN yum install -y \
curl \
gcc \
git \
httpd \
httpd-devel \
libaio \
"nc.x86_64" \
openssl \
openssl-devel \
tar \
unzip \
wget \
zlib-devel \
gcc-c++ \
make
RUN mkdir /rnacentral
RUN mkdir /rnacentral/local
RUN mkdir /rnacentral/static
ENV LOC /rnacentral/local
# Install Python
# NOTE: Python-2.7.11 and python-2.7.11 are DIFFERENT folders, the former contains the sources, the later - binaries
RUN \
cd $LOC && \
curl -OL http://www.python.org/ftp/python/2.7.11/Python-2.7.11.tgz && \
tar -zxvf Python-2.7.11.tgz && \
cd Python-2.7.11 && \
PREFIX=$LOC/python-2.7.11/ && \
export LD_RUN_PATH=$PREFIX/lib && \
./configure --prefix=$PREFIX --enable-shared && \
make && \
make install && \
cd $LOC && \
rm -Rf Python-2.7.11 && \
rm Python-2.7.11.tgz
# Install virtualenv
RUN \
cd $LOC && \
curl -OL https://pypi.python.org/packages/source/v/virtualenv/virtualenv-15.0.0.tar.gz && \
tar -zxvf virtualenv-15.0.0.tar.gz && \
cd virtualenv-15.0.0 && \
$LOC/python-2.7.11/bin/python setup.py install && \
cd $LOC && \
rm -Rf virtualenv-15.0.0.tar.gz && \
rm -Rf virtualenv-15.0.0
# Create RNAcentral virtual environment
RUN \
cd $LOC && \
mkdir virtualenvs && \
cd virtualenvs && \
$LOC/python-2.7.11/bin/virtualenv RNAcentral --python=$LOC/python-2.7.11/bin/python
# Install Redis (the stable version of Redis requires GCC 4.9 or above, not available for centos 6)
RUN \
cd $LOC && \
curl -OL http://download.redis.io/releases/redis-5.0.9.tar.gz && \
tar -xvzf redis-5.0.9.tar.gz && \
cd redis-5.0.9 && \
make && \
cd $LOC && \
rm redis-5.0.9.tar.gz && \
mv redis-5.0.9 redis
# Install libevent (memcached requirement)
RUN \
cd $LOC && \
curl --tlsv1 -OL https://github.com/downloads/libevent/libevent/libevent-2.0.21-stable.tar.gz && \
tar -zxvf libevent-2.0.21-stable.tar.gz && \
cd libevent-2.0.21-stable && \
./configure --prefix=$LOC/libevent && \
make && \
make install && \
cd $LOC && \
rm -Rf libevent-2.0.21-stable && \
rm libevent-2.0.21-stable.tar.gz
# Install memcached (requires libevent)
RUN \
cd $LOC && \
curl -OL http://www.memcached.org/files/memcached-1.4.17.tar.gz && \
tar -zxvf memcached-1.4.17.tar.gz && \
cd memcached-1.4.17 && \
./configure --prefix=$LOC/memcached --with-libevent=$LOC/libevent/ && \
make && \
make install && \
cd $LOC && \
rm -Rf memcached-1.4.17 && \
rm memcached-1.4.17.tar.gz
# Create a user for memcached
RUN adduser -g root xfm_adm
# Install Infernal
RUN \
cd $LOC && \
curl -OL http://eddylab.org/infernal/infernal-1.1.1.tar.gz && \
tar -xvzf infernal-1.1.1.tar.gz && \
cd infernal-1.1.1 && \
./configure --prefix=$LOC/infernal-1.1.1 && \
make && \
make install && \
cd easel && \
make install && \
cd $LOC && \
rm infernal-1.1.1.tar.gz
# Install mod_wsgi
RUN \
cd $LOC && \
mkdir httpd && \
mkdir httpd/modules && \
curl --tlsv1 -L -o mod_wsgi-3.4.tar.gz https://github.com/GrahamDumpleton/mod_wsgi/archive/3.4.tar.gz && \
tar -zxvf mod_wsgi-3.4.tar.gz && \
cd mod_wsgi-3.4 && \
export LD_RUN_PATH=$LOC/python-2.7.11/lib && \
export LD_LIBRARY_PATH=$LOC/python-2.7.11/lib/:$LD_LIBRARY_PATH && \
./configure --with-python=$LOC/python-2.7.11/bin/python && \
make && \
mv .libs/mod_wsgi.so $LOC/httpd/modules && \
cd $LOC && \
rm -Rf mod_wsgi-3.4 && \
rm mod_wsgi-3.4.tar.gz
# Install UCSC utilities
RUN \
cd $LOC && \
mkdir ucsc && \
cd ucsc && \
wget http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/fetchChromSizes && \
wget http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/bedToBigBed && \
chmod o+x fetchChromSizes && \
chmod o+x bedToBigBed
# Define container environment variables
ENV RNACENTRAL_HOME /rnacentral/rnacentral-webcode
ENV RNACENTRAL_LOCAL /rnacentral/local
# Install Django requirements
ADD rnacentral/requirements.txt $RNACENTRAL_HOME/rnacentral/
RUN \
source $LOC/virtualenvs/RNAcentral/bin/activate && \
pip install -r $RNACENTRAL_HOME/rnacentral/requirements.txt
# Install Node.js
RUN curl -sL https://rpm.nodesource.com/setup_6.x | bash -
RUN yum install -y nodejs
# Install NPM dependencies
ADD rnacentral/portal/static/package.json $RNACENTRAL_HOME/rnacentral/portal/static/
RUN cd $RNACENTRAL_HOME/rnacentral/portal/static && npm install --only=production
# Expose a container port where the website is served
EXPOSE 8000
# Start up the app
ENTRYPOINT \
source $LOC/virtualenvs/RNAcentral/bin/activate && \
supervisord -c $RNACENTRAL_HOME/supervisor/supervisor.conf && \
python $RNACENTRAL_HOME/rnacentral/manage.py runserver 0.0.0.0:8000
Dockerfile-development → Dockerfile-local
View file @ 494d6073
#------------------------------------------------------------------------------
#
# This Dockerfile is meant for containerized deployment with Kubernetes.
# This Dockerfile is meant for local development.
#
#-------------------------------------------------------------------------------
FROM debian:buster-slim
FROM python:3.8-slim
RUN apt-get update && apt-get install -y \
g++ \
......@@ -12,19 +12,22 @@ RUN apt-get update && apt-get install -y \
curl \
tar \
git \
python2.7 \
libpython2.7-dev \
python-pip \
vim \
supervisor && \
useradd -m -d /srv/rnacentral -s /bin/bash rnacentral
rm -rf /var/lib/apt/lists/*
ENV RNACENTRAL_HOME=/srv/rnacentral
ENV RNACENTRAL_LOCAL=$RNACENTRAL_HOME/local
ENV SUPERVISOR_CONF_DIR=${SUPERVISOR_CONF_DIR:-"/srv/rnacentral/supervisor"}
ENV RNACENTRAL_LOCAL=/srv/rnacentral/local
ENV SUPERVISOR_CONF_DIR=/srv/rnacentral/supervisor
# Create folders. Install Infernal and node.js
# Create folders
RUN \
mkdir -p $RNACENTRAL_LOCAL && \
mkdir -p $SUPERVISOR_CONF_DIR && \
mkdir /srv/rnacentral/log && \
mkdir /srv/rnacentral/static
# Install Infernal and node.js
RUN \
mkdir -p $RNACENTRAL_HOME/local && \
cd $RNACENTRAL_LOCAL && \
curl -OL http://eddylab.org/infernal/infernal-1.1.1.tar.gz && \
tar -xvzf infernal-1.1.1.tar.gz && \
......@@ -36,23 +39,28 @@ RUN \
make install && \
cd $RNACENTRAL_LOCAL && \
rm infernal-1.1.1.tar.gz && \
curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
curl -sL https://deb.nodesource.com/setup_lts.x | bash - && \
apt-get install -y nodejs
USER rnacentral
# Set work directory
ENV RNACENTRAL_HOME=/srv/rnacentral/rnacentral-webcode
RUN mkdir -p $RNACENTRAL_HOME
WORKDIR $RNACENTRAL_HOME
# Download RNAcentral, install requirements and node.js dependencies
RUN \
cd $RNACENTRAL_HOME && \
git clone https://github.com/RNAcentral/rnacentral-webcode.git && \
pip install -r $RNACENTRAL_HOME/rnacentral-webcode/rnacentral/requirements.txt && \
pip install gunicorn && \
cd $RNACENTRAL_HOME/rnacentral-webcode/rnacentral/portal/static && npm install --only=production
# Copy requirements
COPY rnacentral/requirements* .
# Install requirements
RUN pip3 install -r requirements.txt && pip3 install -r requirements_dev.txt
COPY ./entrypoint.sh /entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
# Install NPM dependencies
COPY rnacentral/portal/static/package.json /srv/rnacentral
RUN cd /srv/rnacentral && npm install --only=production
# Expose a container port where the website is served
EXPOSE 8000
# Copy openssl.cnf
COPY openssl/openssl.cnf /etc/ssl/openssl.cnf
CMD [ "/bin/sh", "-c", "/usr/bin/supervisord -c ${SUPERVISOR_CONF_DIR}/supervisord.conf" ]
# Run entrypoint
COPY ./entrypoint-local.sh /
RUN chmod 744 /entrypoint-local.sh
ENTRYPOINT ["/entrypoint-local.sh"]
Dockerfile-production deleted 100644 → 0
View file @ 8103aca9
#------------------------------------------------------------------------------
# We have 2 ways to deploy RNAcentral in production:
# bare-metal managed (no root permissions) CentOS 6 VMs and Kubernetes.
#
# This Dockerfile is meant for containerized deployment with Kubernetes.
#
#-------------------------------------------------------------------------------
FROM debian:latest
RUN apt-get update && apt-get install -y \
g++ \
build-essential \
curl \
wget \
tar \
git \
python2.7 \
libpython2.7-dev \
python-pip \
redis-server \
memcached
ADD . /srv/rnacentral
RUN mkdir /srv/rnacentral/local
RUN mkdir /srv/rnacentral/static
ENV RNACENTRAL_LOCAL /srv/rnacentral/local
ENV RNACENTRAL_HOME /srv/rnacentral
# Install Infernal
RUN \
cd $RNACENTRAL_LOCAL && \
curl -OL http://eddylab.org/infernal/infernal-1.1.1.tar.gz && \
tar -xvzf infernal-1.1.1.tar.gz && \
cd infernal-1.1.1 && \
./configure --prefix=$RNACENTRAL_LOCAL/infernal-1.1.1 && \
make && \
make install && \
cd easel && \
make install && \
cd $RNACENTRAL_LOCAL && \
rm infernal-1.1.1.tar.gz
# Install Django requirements
RUN pip install -r /srv/rnacentral/rnacentral/requirements.txt
# Install node.js
RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
RUN apt-get install -y nodejs
# Install node.js dependencies
ADD rnacentral/portal/static/package.json $RNACENTRAL_HOME/rnacentral/portal/static/
RUN cd $RNACENTRAL_HOME/rnacentral/portal/static && npm install --only=production
# Expose a container port where the website is served
EXPOSE 8000
# Start up the app
CMD python /srv/rnacentral/rnacentral/manage.py runserver 0.0.0.0:8000
Dockerfile-rq 0 → 100644
View file @ 494d6073
#------------------------------------------------------------------------------
#
# This Dockerfile is for installing rq-dashboard and monitoring RQ queues.
#
#-------------------------------------------------------------------------------
FROM python:3.8-slim
RUN pip install rq-dashboard
RUN pip install click==7.1.2 # rq-dashboard==0.6.3 is incompatible with click==8.0.1
README.md
View file @ 494d6073
......@@ -19,14 +19,25 @@ The development of RNAcentral is coordinated by the
git clone --recursive https://github.com/RNAcentral/rnacentral-webcode.git
```
2. Edit database connection parameters in `rnacentral/local_settings.py`. Use the [public postgres database](https://rnacentral.org/help/public-database) when developing outside the EBI network.
2. Using the `.env-example` file as a template, create the `.env` file and change the environment variables as desired.
The [public postgres database](https://rnacentral.org/help/public-database) is configured by default.
If you want to use another database, add the following environment variables: `DB_HOST`, `DB_NAME`, `DB_USER`,
`DB_PASSWORD`, and `DB_PORT`.
3. Run the app using [Docker](https://www.docker.com):
3. For Mac OS users, install [mutagen](https://mutagen.io/) (Docker on Mac has some performance issues)
```
export RNACENTRAL_HOME=/path/to/rnacentral/code
cd $RNACENTRAL_HOME
docker-compose up --build
brew install mutagen-io/mutagen/mutagen-beta
```
4. Run the app using [Docker](https://www.docker.com):
```
For Linux users:
docker-compose -f docker-compose.local.yml up --build
For Mac OS users:
mutagen compose -f docker-compose.local.mac.yml up --build
```
## Feedback
......
docker-compose.local.mac.yml 0 → 100644
View file @ 494d6073
# Set the Docker Compose file format
version: '3.7'
# Define the services (containers) that make up the project
services:
# RNAcentral website
rnacentral:
build:
context: .
dockerfile: Dockerfile-local
environment:
- DB_HOST=${DB_HOST}
- DB_NAME=${DB_NAME}
- DB_USER=${DB_USER}
- DB_PASSWORD=${DB_PASSWORD}
- DB_PORT=${DB_PORT}
- SECRET_KEY=${SECRET_KEY}
- DJANGO_DEBUG=${DJANGO_DEBUG}
- S3_HOST=${S3_HOST}
- S3_KEY=${S3_KEY}
- S3_SECRET=${S3_SECRET}
- EBI_SEARCH_ENDPOINT=${EBI_SEARCH_ENDPOINT}
command: python rnacentral/manage.py runserver 0.0.0.0:8000
ports:
- 8000:8000
volumes:
- rnacentral:/srv/rnacentral/rnacentral-webcode
- .env:/srv/rnacentral/rnacentral-webcode/.env
depends_on:
- redis
networks:
- redis-network
- memcached-network
# rqworker - used to export search results
rqworker:
build:
context: .
dockerfile: Dockerfile-local
environment:
- DB_HOST=${DB_HOST}
- DB_NAME=${DB_NAME}
- DB_USER=${DB_USER}
- DB_PASSWORD=${DB_PASSWORD}
- DB_PORT=${DB_PORT}
- SECRET_KEY=${SECRET_KEY}
- DJANGO_DEBUG=${DJANGO_DEBUG}
- S3_HOST=${S3_HOST}
- S3_KEY=${S3_KEY}
- S3_SECRET=${S3_SECRET}
- EBI_SEARCH_ENDPOINT=${EBI_SEARCH_ENDPOINT}