Commit 613bfd8b authored by carlosribas's avatar carlosribas
Browse files

Remove old pipelines

parent b0f52660
node {
stopKibana()
stopFilebeatPG()
stopFilebeatOY()
stopLogstash()
stopElasticsearch()
startElasticsearch()
startLogstash()
startFilebeatOY()
startFilebeatPG()
startKibana()
}
private void stopLogstash() {
stage("Stopping logstash") {
try {
sh '''kill $(ps aux | grep logstash | awk '{print $2}')'''
} catch (err) {
echo err.toString()
}
sleep 3
}
}
private void startLogstash() {
stage("Starting logstash") {
dir("/net/isilonP/public/rw/homes/xfm_adm/src") {
sh 'BUILD_ID=dontKillMe nohup logstash-5.4.3/bin/logstash -f logstash-apache.conf &'
sleep 10
}
}
}
private void stopElasticsearch() {
stage("Stopping elasticsearch") {
try {
sh '''kill $(ps aux | grep 'elasticsearch' | awk '{print $2}')'''
} catch(err) {
echo err.toString()
}
sleep 3
}
}
private void startElasticsearch() {
stage("Starting elasticsearch") {
dir("/net/isilonP/public/rw/homes/xfm_adm/src") {
sh '''
BUILD_ID=dontKillMe nohup elasticsearch-5.4.3/bin/elasticsearch -Enetwork.host=0.0.0.0 -Ebootstrap.system_call_filter=false &
'''
sleep 10
}
}
}
private void stopKibana() {
stage("Stopping kibana") {
try {
sh '''kill $(ps aux | grep 'elasticsearch' | awk '{print $2}')'''
} catch(err) {
echo err.toString()
}
sleep 3
}
}
private void startKibana() {
stage("Starting kibana") {
dir("/net/isilonP/public/rw/homes/xfm_adm/src") {
sh 'BUILD_ID=dontKillMe nohup kibana-5.4.3-linux-x86_64/bin/kibana -H 0.0.0.0 &'
sleep 10
}
}
}
private void stopFilebeatOY() {
stage("Stopping filebeat on OY") {
try {
sh '''kill $(ps aux | grep 'filebeat' | awk '{print $2}')'''
} catch(err) {
echo err.toString()
}
}
}
private void startFilebeatOY() {
stage("Starting filebeat on OY") {
dir("/net/isilonP/public/rw/homes/xfm_adm/src") {
sh 'BUILD_ID=dontKillMe nohup ./filebeat-5.5.0-linux-x86_64/filebeat -c filebeat.yml &'
sleep 10
}
}
}
private void stopFilebeatPG() {
stage("Stopping filebeat on PG") {
try {
sh '''
ssh ves-pg-a4 "ps aux | grep filebeat | awk '{print \\$2}' | xargs kill"
'''
} catch(err) {
echo err.toString()
}
}
}
private void startFilebeatPG() {
stage("Starting filebeat on PG") {
sh '''
ssh ves-pg-a4 'DISPLAY=:0 nohup /net/isilonP/public/rw/homes/xfm_adm/bin/filebeat-5.5.0-linux-x86_64/filebeat -c /net/isilonP/public/rw/homes/xfm_adm/bin/filebeat.yml < /dev/null > std.out 2> std.err &'
'''
sleep 10
}
}
node {
start_supervisor()
}
private void start_supervisor() {
sh '''
cd /nfs/public/rw/xfam/rnacentral/live
source local/virtualenvs/RNAcentral/bin/activate
cd rnacentral-webcode/rnacentral
source scripts/env.sh
fab -H ves-oy-a4 production start_supervisor
'''
}
node {
start_supervisor()
}
private void start_supervisor() {
sh '''
cd /nfs/public/rw/xfam/rnacentral/live
source local/virtualenvs/RNAcentral/bin/activate
cd rnacentral-webcode/rnacentral
source scripts/env.sh
fab -H ves-pg-a4 production start_supervisor
'''
}
pipeline {
agent any
parameters {
choice(
name: 'DATABASE',
choices: ['FB-PG10', 'HH-PG10', 'PRO-PG10', 'DEV-PG10', 'TEST-PG10'],
description: 'Which database instance to use?'
)
choice(
name: 'EBI_SEARCH_ENDPOINT',
choices: ['PRO', 'DEV'],
description: 'Which ebi search index to use?'
)
choice(
name: 'RQ_QUEUE',
choices: ['OY', 'LOCAL', 'PG'],
description: 'Which redis queues settings to use?'
)
choice(
name: 'COMPRESSOR',
choices: ['True', 'False'],
description: 'Shall we use django-compressor?'
)
choice(
name: 'QUICK',
choices: ['True', 'False'],
description: 'Fast rebuild: pull, collect (and compress) static, touch wsgi.py.'
)
gitParameter(
branchFilter: 'origin/(.*)',
defaultValue: 'master',
name: 'BRANCH',
type: 'PT_BRANCH',
description: 'Name of the branch to test and deploy'
)
string(
name: 'MESSAGE',
defaultValue: '',
description: 'Message to show on the website'
)
}
stages {
stage("deploy") {
environment {
DATABASE_PRO_PG10 = credentials('database_pro_pg10.py')
DATABASE_DEV_PG10 = credentials('database_dev_pg10.py')
DATABASE_TEST_PG10 = credentials('database_test_pg10.py')
DATABASE_FB_PG10 = credentials('database_fb_pg10.py')
DATABASE_HH_PG10 = credentials('database_hh_pg10.py')
EBI_SEARCH_ENDPOINT_PRO = credentials('ebi_search_endpoint_pro.py')
EBI_SEARCH_ENDPOINT_DEV = credentials('ebi_search_endpoint_dev.py')
RQ_QUEUE_LOCAL = credentials('rq_queue_local.py')
RQ_QUEUE_OY = credentials('rq_queue_oy.py')
RQ_QUEUE_PG = credentials('rq_queue_pg.py')
LOCAL_SETTINGS = credentials('local_settings.py')
ENV_SH = credentials('env_sh')
}
steps {
script {
sh 'rm -f /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/databases.py'
if (params.DATABASE == 'PRO-PG10') {
sh 'cat ${DATABASE_PRO_PG10} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/databases.py'
} else if (params.DATABASE == 'DEV-PG10') {
sh 'cat ${DATABASE_DEV_PG10} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/databases.py'
} else if (params.DATABASE == 'TEST-PG10') {
sh 'cat ${DATABASE_TEST_PG10} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/databases.py'
} else if (params.DATABASE == 'FB-PG10') {
sh 'cat ${DATABASE_FB_PG10} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/databases.py'
} else if (params.DATABASE == 'HH-PG10') {
sh 'cat ${DATABASE_HH_PG10} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/databases.py'
}
}
script {
sh 'rm -f /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/ebi_search_endpoints.py'
if (params.EBI_SEARCH_ENDPOINT == 'PRO') {
sh 'cat ${EBI_SEARCH_ENDPOINT_PRO} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/ebi_search_endpoints.py'
} else if (params.EBI_SEARCH_ENDPOINT == 'DEV') {
sh 'cat ${EBI_SEARCH_ENDPOINT_DEV} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/ebi_search_endpoints.py'
}
}
script {
sh 'rm -f /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/rq_queues.py'
if (params.RQ_QUEUE == 'LOCAL') {
sh 'cat ${RQ_QUEUE_LOCAL} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/rq_queues.py'
} else if (params.RQ_QUEUE == 'OY') {
sh 'cat ${RQ_QUEUE_OY} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/rq_queues.py'
} else if (params.RQ_QUEUE == 'PG') {
sh 'cat ${RQ_QUEUE_PG} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/rq_queues.py'
}
}
script {
sh 'rm -f /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/compressor.py'
if (params.COMPRESSOR == 'True') {
sh 'echo "COMPRESS_ENABLED = True" > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/compressor.py'
} else if (params.COMPRESSOR == 'False') {
sh 'echo "COMPRESS_ENABLED = False" > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/compressor.py'
}
}
script {
sh 'rm -f /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/portal/templates/portal/message'
if (params.MESSAGE) {
sh "echo ${params.MESSAGE} > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/portal/templates/portal/message"
}
}
sh 'rm -f /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/local_settings.py'
sh 'rm -f /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/local_settings.pyc'
sh 'cat $LOCAL_SETTINGS > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/local_settings.py'
sh 'rm -f /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/env.sh'
sh 'cat $ENV_SH > /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/env.sh'
sh """
cd /nfs/public/rw/xfam/rnacentral/live
source local/virtualenvs/RNAcentral/bin/activate
cd rnacentral-webcode/rnacentral
source scripts/env.sh
# update repository to fetch the latest fabfile
git reset --hard
git fetch --all
git checkout ${params.BRANCH}
git pull
# update pip dependencies, if deployment is not quick;
# otherwise fabfile imports django settings,
# fails to import python dependencies and crashes
if [ "${params.QUICK}" == "False" ]; then
pip install --upgrade -r requirements.txt
fi
# update the embeddable component of the sequence search
cd /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/sequence_search/static/rnacentral-sequence-search-embed
git pull
fab -H ves-oy-a4 production deploy:git_branch=${params.BRANCH},quick=${params.QUICK},compress=${params.COMPRESSOR}
"""
}
}
}
}
pipeline {
agent any
parameters {
choice(
name: 'DATABASE',
choices: ['HH-PG10', 'FB-PG10', 'PRO-PG10', 'DEV-PG10', 'TEST-PG10'],
description: 'Which database instance to use?'
)
choice(
name: 'EBI_SEARCH_ENDPOINT',
choices: ['PRO', 'DEV'],
description: 'Which ebi search index to use?'
)
choice(
name: 'RQ_QUEUE',
choices: ['PG', 'OY', 'LOCAL'],
description: 'Which redis queues settings to use?'
)
choice(
name: 'COMPRESSOR',
choices: ['True', 'False'],
description: 'Shall we use django-compressor?'
)
choice(
name: 'QUICK',
choices: ['True', 'False'],
description: 'Fast rebuild: pull, collect (and compress) static, touch wsgi.py.'
)
gitParameter(
branchFilter: 'origin/(.*)',
defaultValue: 'master',
name: 'BRANCH',
type: 'PT_BRANCH',
description: 'Name of the branch to test and deploy'
)
string(
name: 'MESSAGE',
defaultValue: '',
description: 'Message to show on the website'
)
}
stages {
stage("deploy") {
environment {
DATABASE_PRO_PG10 = credentials('database_pro_pg10.py')
DATABASE_DEV_PG10 = credentials('database_dev_pg10.py')
DATABASE_TEST_PG10 = credentials('database_test_pg10.py')
DATABASE_FB_PG10 = credentials('database_fb_pg10.py')
DATABASE_HH_PG10 = credentials('database_hh_pg10.py')
EBI_SEARCH_ENDPOINT_PRO = credentials('ebi_search_endpoint_pro.py')
EBI_SEARCH_ENDPOINT_DEV = credentials('ebi_search_endpoint_dev.py')
RQ_QUEUE_LOCAL = credentials('rq_queue_local.py')
RQ_QUEUE_OY = credentials('rq_queue_oy.py')
RQ_QUEUE_PG = credentials('rq_queue_pg.py')
LOCAL_SETTINGS = credentials('local_settings.py')
ENV_SH = credentials('env_sh')
}
steps {
script {
if (params.DATABASE == 'PRO-PG10') {
sh 'cat ${DATABASE_PRO_PG10} > tmp'
} else if (params.DATABASE == 'DEV-PG10') {
sh 'cat ${DATABASE_DEV_PG10} > tmp'
} else if (params.DATABASE == 'TEST-PG10') {
sh 'cat ${DATABASE_TEST_PG10} > tmp'
} else if (params.DATABASE == 'FB-PG10') {
sh 'cat ${DATABASE_FB_PG10} > tmp'
} else if (params.DATABASE == 'HH-PG10') {
sh 'cat ${DATABASE_HH_PG10} > tmp'
}
sh 'rsync -av tmp ves-pg-a4:/nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/databases.py'
}
script {
if (params.EBI_SEARCH_ENDPOINT == 'PRO') {
sh 'cat ${EBI_SEARCH_ENDPOINT_PRO} > tmp'
} else if (params.EBI_SEARCH_ENDPOINT == 'DEV') {
sh 'cat ${EBI_SEARCH_ENDPOINT_DEV} > tmp'
}
sh 'rsync -av tmp ves-pg-a4:/nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/ebi_search_endpoints.py'
}
script {
if (params.RQ_QUEUE == 'LOCAL') {
sh 'cat ${RQ_QUEUE_LOCAL} > tmp'
} else if (params.RQ_QUEUE == 'OY') {
sh 'cat ${RQ_QUEUE_OY} > tmp'
} else if (params.RQ_QUEUE == 'PG') {
sh 'cat ${RQ_QUEUE_PG} > tmp'
}
sh 'rsync -av tmp ves-pg-a4:/nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/rq_queues.py'
}
script {
if (params.COMPRESSOR == 'True') {
sh 'echo "COMPRESS_ENABLED = True" > tmp'
} else if (params.COMPRESSOR == 'False') {
sh 'echo "COMPRESS_ENABLED = False" > tmp'
}
sh 'rsync -av tmp ves-pg-a4:/nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/compressor.py'
}
script {
sh 'rm -f ves-pg-a4:/nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/portal/templates/portal/message'
if (params.MESSAGE) {
sh "echo ${params.MESSAGE} > ves-pg-a4:/nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/portal/templates/portal/message"
}
}
sh 'cat $LOCAL_SETTINGS > tmp'
sh 'rsync -av tmp ves-pg-a4:/nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/local_settings.py'
sh 'cat $ENV_SH > tmp'
sh 'rsync -av tmp ves-pg-a4:/nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/rnacentral/env.sh'
sh """
cd /nfs/public/rw/xfam/rnacentral/live
source local/virtualenvs/RNAcentral/bin/activate
cd rnacentral-webcode/rnacentral
source scripts/env.sh
# update repository to fetch the latest fabfile
git reset --hard
git fetch --all
git checkout ${params.BRANCH}
git pull
# update the embeddable component of the sequence search
cd /nfs/public/rw/xfam/rnacentral/live/rnacentral-webcode/rnacentral/sequence_search/static/rnacentral-sequence-search-embed
git pull
fab -H ves-pg-a4 production deploy:git_branch=${params.BRANCH},quick=${params.QUICK},compress=${params.COMPRESSOR}
"""
}
}
}
}
pipeline {
agent any
stages {
stage("Copy local settings to Jenkins environment") {
environment {
LOCAL_SETTINGS = credentials('local_settings.py')
DATABASE = credentials('database_pro.py')
EBI_SEARCH_ENDPOINT = credentials('ebi_search_endpoint_pro.py')
RQ_QUEUE = credentials('rq_queue_pg.py')
COMPRESSOR = credentials('compressor_pro.py')
}
steps {
sh 'cat $LOCAL_SETTINGS > rnacentral/rnacentral/local_settings.py'
sh 'cat $DATABASE > rnacentral/rnacentral/databases.py'
sh 'cat $EBI_SEARCH_ENDPOINT > rnacentral/rnacentral/ebi_search_endpoints.py'
sh 'cat $RQ_QUEUE > rnacentral/rnacentral/rq_queues.py'
sh 'cat $COMPRESSOR > rnacentral/rnacentral/compressor.py'
}
}
stage("refresh DEV from last snapshot") {
environment {
USERPASSWORD = credentials('refresh_vdbs_login')
USER = "${env.USERPASSWORD_USR}"
PASSWORD = "${env.USERPASSWORD_PSW}"
}
steps {
sh '''
cd rnacentral
fab ebi_cli --user=$USER --password=$PASSWORD refresh_dev
'''
}
}
}
}
\ No newline at end of file
// escaping and interpolation between Jenkinsfile and bash:
// https://jenkins.io/doc/book/pipeline/syntax/
// https://github.com/puzzle/jenkins-techlab/blob/master/labs/05_string_interpolation_quoting_escaping.md
// https://gist.github.com/Faheetah/e11bd0315c34ed32e681616e41279ef4
// https://stackoverflow.com/questions/41539076/how-to-pass-variables-from-jenkinsfile-to-shell-command
// handling credentials:
// https://stackoverflow.com/questions/47475160/how-to-use-multiple-credentials-in-withcredentials-in-jenkins-pipeline
// Jenkins environment variable:
// https://jenkins.io/doc/book/pipeline/jenkinsfile/#working-with-the-environment
pipeline {
agent any
parameters {
choice(
name: 'DATABASE',
choices: ['FB1', 'PG', 'BOTH'],
description: 'Which database instance to use?'
)
}
stages {
stage("Copy local settings to Jenkins environment") {
// This is really stupid, but otherwise Django would miss settings and wouldn't boot
environment {
LOCAL_SETTINGS = credentials('local_settings.py')
DATABASE = credentials('database_pro.py')
EBI_SEARCH_ENDPOINT = credentials('ebi_search_endpoint_pro.py')
RQ_QUEUE = credentials('rq_queue_pg.py')
COMPRESSOR = credentials('compressor_pro.py')
}
steps {
sh 'cat $LOCAL_SETTINGS > rnacentral/rnacentral/local_settings.py'
sh 'cat $DATABASE > rnacentral/rnacentral/databases.py'
sh 'cat $EBI_SEARCH_ENDPOINT > rnacentral/rnacentral/ebi_search_endpoints.py'
sh 'cat $RQ_QUEUE > rnacentral/rnacentral/rq_queues.py'
sh 'cat $COMPRESSOR > rnacentral/rnacentral/compressor.py'
}
}
stage("FB1 refresh") {
when {
expression { params.DATABASE == 'FB1' || params.DATABASE == 'BOTH' }
}
environment {
USERPASSWORD = credentials('refresh_vdbs_login')
USER = "${env.USERPASSWORD_USR}"
PASSWORD = "${env.USERPASSWORD_PSW}"
}
steps {
// Note that we're using the virtualenv from .rnacentral-live virtualenv,
// which might get a bit crazy, but this is just for fabric python package.
sh '''
pushd /nfs/public/rw/xfam/rnacentral/live
source local/virtualenvs/RNAcentral/bin/activate
popd
cd rnacentral
fab fb1 --user=$USER --password=$PASSWORD refresh_fb1
'''
}
}
stage("Push replication from FB1 to PG") {
environment {
USERPASSWORD = credentials('refresh_vdbs_login')
USER = "${env.USERPASSWORD_USR}"
PASSWORD = "${env.USERPASSWORD_PSW}"
}
steps {
// always run this stage
sh '''
pushd /nfs/public/rw/xfam/rnacentral/live
source local/virtualenvs/RNAcentral/bin/activate
popd
cd rnacentral
fab fb1 --user=$USER --password=$PASSWORD push_replication
'''
}
}
stage("PG refresh") {
when {
expression { params.DATABASE == 'PG' || params.DATABASE == 'BOTH' }
}
environment {
USERPASSWORD = credentials('refresh_vdbs_login')
USER = "${env.USERPASSWORD_USR}"
PASSWORD = "${env.USERPASSWORD_PSW}"
}
steps {
sh '''
pushd /nfs/public/rw/xfam/rnacentral/live
source local/virtualenvs/RNAcentral/bin/activate
popd
cd rnacentral
fab pg --user=$USER --password=$PASSWORD refresh_pg
'''
}
}
}
}
pipeline {
agent any
stages {
stage("Copy local settings to Jenkins environment") {
environment {
LOCAL_SETTINGS = credentials('local_settings.py')
DATABASE = credentials('database_pro.py')
EBI_SEARCH_ENDPOINT = credentials('ebi_search_endpoint_pro.py')
RQ_QUEUE = credentials('rq_queue_pg.py')
COMPRESSOR = credentials('compressor_pro.py')
}
steps {
sh 'cat $LOCAL_SETTINGS > rnacentral/rnacentral/local_settings.py'
sh 'cat $DATABASE > rnacentral/rnacentral/databases.py'
sh 'cat $EBI_SEARCH_ENDPOINT > rnacentral/rnacentral/ebi_search_endpoints.py'
sh 'cat $RQ_QUEUE > rnacentral/rnacentral/rq_queues.py'
sh 'cat $COMPRESSOR > rnacentral/rnacentral/compressor.py'
}
}
stage("refresh TST from last snapshot") {
environment {
USERPASSWORD = credentials('refresh_vdbs_login')
USER = "${env.USERPASSWORD_USR}"
PASSWORD = "${env.USERPASSWORD_PSW}"
}
steps {
sh '''
cd rnacentral
fab ebi_cli --user=$USER --password=$PASSWORD refresh_tst
'''
}
}
}
}
\ No newline at end of file
node {
deploy()
}
private void deploy() {
sh '''
cd /nfs/public/rw/xfam/rnacentral/live
source local/virtualenvs/RNAcentral/bin/activate
cd rnacentral-webcode/rnacentral
source scripts/env.sh
fab localhost create_sitemaps
fab -H ves-oy-a4 production rsync_sitemaps:remote_host=ves-pg-a4
'''
}
// ---------------------------------------------------------------------------------------------------------------------
//
// There's a caveat in this Jenkinsfile:
// Its build and unit-test steps are running in Jenkins environment and are
// using the branch that is configured in Jenkins interface,
// while deploy step is running in the filesystem environment and checks out and updates that branch.
//
// ---------------------------------------------------------------------------------------------------------------------
pipeline {
agent any
parameters {
choice(
name: 'DATABASE',
choices: ['PRO-PG10', 'DEV-PG10', 'TEST-PG10', 'FB-PG10', 'HH-PG10'],
description: 'Which database instance to use?'
)