Commit 17ba48e7 authored by David Mendez's avatar David Mendez

Merge branch 'staging' into 'master'

Add CI to Deploy to staging

See merge request !1
parents a8618220 84a9bca4
Pipeline #93282 passed with stages
in 4 minutes and 4 seconds
......@@ -2,8 +2,10 @@ variables:
DOCKER_TLS_CERTDIR: ""
GIT_STRATEGY: clone
REGISTRY_USER: chembl/chembl
SERVER_APPLICATION: main-web-interface/es-subset-generator
SERVER_APPLICATION: main-web-interface/es-subset-generator/es-subset-generator-api
TASKS_APPLICATION: main-web-interface/es-subset-generator/es-subset-generator-tasks-daemon
SERVER_IMAGE_TAG: ${CI_REGISTRY}/${REGISTRY_USER}/${SERVER_APPLICATION}:${CI_COMMIT_SHORT_SHA}
TASKS_DAEMON_IMAGE_TAG: ${CI_REGISTRY}/${REGISTRY_USER}/${TASKS_APPLICATION}:${CI_COMMIT_SHORT_SHA}
DOCKER_DRIVER: overlay
CONFIGS_FOLDER: run_config
# Config unit tests
......@@ -15,7 +17,6 @@ variables:
# Config prod_hx
CONFIG_PROD_HX_FILE_PATH: ${CONFIGS_FOLDER}/prod_hx.yml
CONFIG_GUNICORN_PROD_HX_FILE_PATH: ${CONFIGS_FOLDER}/prod_hx.gunicorn.config.py
INGRESS_PROD_HX_FILE_PATH: ${CONFIGS_FOLDER}/ingress-prod_hx.yml
AUTOSCALER_PROD_HX_FILE_PATH: ${CONFIGS_FOLDER}/autoscaler-prod_hx.yml
# Config prod_hh
CONFIG_PROD_HH_FILE_PATH: ${CONFIGS_FOLDER}/prod_hh.yml
......@@ -33,9 +34,9 @@ stages:
- prepare_config_tests
- test_and_qa
- prepare_config_staging
# - deploy_to_staging
# - test_staging
# - prepare_config_hx
- deploy_to_staging
- test_staging
- prepare_config_hx
# - deploy_to_prod_hx
# - test_prod_hx
# - prepare_config_hh
......@@ -47,7 +48,7 @@ stages:
# ----------------------------------------------------------------------------------------------------------------------
# build
# ----------------------------------------------------------------------------------------------------------------------
build_docker_image:
build_docker_image_api_server:
image:
name: docker/compose:1.24.1
entrypoint: ["/bin/sh", "-c"]
......@@ -60,11 +61,27 @@ build_docker_image:
- set -x
- docker version # verify docker cli is there. Also prints server info
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- echo building $SERVER_IMAGE_TAG
- echo building ${SERVER_IMAGE_TAG}
- docker-compose build es-subset-generator-prod
- echo Deploying $SERVER_IMAGE_TAG
- docker push $SERVER_IMAGE_TAG
build_docker_image_tasks_daemon:
image:
name: docker/compose:1.24.1
entrypoint: ["/bin/sh", "-c"]
except:
- schedules
stage: build
variables:
DOCKER_HOST: tcp://docker:2375
script:
- set -x
- docker version # verify docker cli is there. Also prints server info
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- echo building ${TASKS_DAEMON_IMAGE_TAG}
- docker-compose build task-status-daemon-prod
- docker push ${TASKS_DAEMON_IMAGE_TAG}
## ----------------------------------------------------------------------------------------------------------------------
## QA
## ----------------------------------------------------------------------------------------------------------------------
......@@ -128,92 +145,88 @@ prepare_config_staging:
- ${CONFIG_STAGING_FILE_PATH}
- ${CONFIG_GUNICORN_STAGING_FILE_PATH}
- ${AUTOSCALER_STAGING_FILE_PATH}
deploy_to_staging:
stage: deploy_to_staging
image:
name: lachlanevenson/k8s-kubectl:latest
entrypoint: ["/bin/sh", "-c"]
only:
- staging
except:
- schedules
environment:
name: staging
script:
- set -x
- echo "$KUBE_CA_PEM_HX" > "$(pwd)/kube.ca.pem"
- kubectl config set-cluster ${KUBE_CLUS_NAME_HX} --server="${KUBE_URL_HX}" --certificate-authority="$(pwd)/kube.ca.pem"
- kubectl config set-credentials ${KUBE_USER_HX} --token="${KUBE_TOKEN_HX}"
- kubectl config set-context ${CHEMBL_NS_STAGING} --cluster=${KUBE_CLUS_NAME_HX} --user=${KUBE_USER_HX}
- kubectl config use-context ${CHEMBL_NS_STAGING}
- echo ${CHEMBL_NS_STAGING} ${KUBE_URL_HX} ${KUBE_CLUS_NAME_HX} ${KUBE_USER_HX}
- kubectl get pods -n ${CHEMBL_NS_STAGING}
- kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_STAGING} --from-file=RUN_CONFIG.yml=${CONFIG_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run=client | kubectl replace -f -
- kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_STAGING} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run=client | kubectl replace -f -
- cat ${CONFIG_STAGING_FILE_PATH}
- cat ${CONFIG_GUNICORN_STAGING_FILE_PATH}
- sed -i "s~<NAMESPACE>~${CHEMBL_NS_STAGING}~" k8s-deployment.yaml
- sed -i "s~<NUM_REPLICAS>~${NUM_REPLICAS_STAGING}~" k8s-deployment.yaml
- sed -i "s~<HTTP_PROXY>~${HTTP_PROXY_HX}~g" k8s-deployment.yaml
- sed -i "s~<HTTPS_PROXY>~${HTTPS_PROXY_HX}~g" k8s-deployment.yaml
- sed -i "s~<NO_PROXY>~${NO_PROXY_HX_DEFAULT}~g" k8s-deployment.yaml
- sed -i "s~<CPU_LIMIT>~${CPU_LIMIT_STAGING}~" k8s-deployment.yaml
- sed -i "s~<CPU_REQUESTED>~${CPU_REQUESTED_STAGING}~" k8s-deployment.yaml
- sed -i "s~<SERVER_IMAGE_TAG>~${SERVER_IMAGE_TAG}~" k8s-deployment.yaml
- sed -i "s~<TASKS_DAEMON_IMAGE_TAG>~${TASKS_DAEMON_IMAGE_TAG}~" k8s-deployment.yaml
- sed -i "s~<RUN_CONFIG_SECRET_NAME>~${RUN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
- sed -i "s~<GUNICORN_CONFIG_SECRET_NAME>~${GUNICORN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
- cat k8s-deployment.yaml
- kubectl apply -n ${CHEMBL_NS_STAGING} -f k8s-deployment.yaml
- kubectl get pods -n ${CHEMBL_NS_STAGING}
- cat ${AUTOSCALER_STAGING_FILE_PATH}
- kubectl apply -n ${CHEMBL_NS_STAGING} -f ${AUTOSCALER_STAGING_FILE_PATH}
#
#deploy_to_staging:
# stage: deploy_to_staging
# image:
# name: lachlanevenson/k8s-kubectl:latest
# entrypoint: ["/bin/sh", "-c"]
# only:
# - staging
# except:
# - schedules
# environment:
# name: staging
# url: ${SERVER_URL_STAGING}
# script:
# - set -x
# - echo "$KUBE_CA_PEM_HX" > "$(pwd)/kube.ca.pem"
# - kubectl config set-cluster ${KUBE_CLUS_NAME_HX} --server="${KUBE_URL_HX}" --certificate-authority="$(pwd)/kube.ca.pem"
# - kubectl config set-credentials ${KUBE_USER_HX} --token="${KUBE_TOKEN_HX}"
# - kubectl config set-context ${CHEMBL_NS_STAGING} --cluster=${KUBE_CLUS_NAME_HX} --user=${KUBE_USER_HX}
# - kubectl config use-context ${CHEMBL_NS_STAGING}
# - echo ${CHEMBL_NS_STAGING} ${KUBE_URL_HX} ${KUBE_CLUS_NAME_HX} ${KUBE_USER_HX}
# - kubectl get pods -n ${CHEMBL_NS_STAGING}
# - kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_STAGING} --from-file=RUN_CONFIG.yml=${CONFIG_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run=client | kubectl replace -f -
# - kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_STAGING} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run=client | kubectl replace -f -
# - cat ${CONFIG_STAGING_FILE_PATH}
# - cat ${CONFIG_GUNICORN_STAGING_FILE_PATH}
# - sed -i "s~<NAMESPACE>~${CHEMBL_NS_STAGING}~" k8s-deployment.yaml
# - sed -i "s~<NUM_REPLICAS>~${NUM_REPLICAS_STAGING}~" k8s-deployment.yaml
# - sed -i "s~<HTTP_PROXY>~${HTTP_PROXY_HX}~g" k8s-deployment.yaml
# - sed -i "s~<HTTPS_PROXY>~${HTTPS_PROXY_HX}~g" k8s-deployment.yaml
# - sed -i "s~<NO_PROXY>~${NO_PROXY_HX_DEFAULT}~g" k8s-deployment.yaml
# - sed -i "s~<CPU_LIMIT>~${CPU_LIMIT_STAGING}~" k8s-deployment.yaml
# - sed -i "s~<CPU_REQUESTED>~${CPU_REQUESTED_STAGING}~" k8s-deployment.yaml
# - sed -i "s~<SERVER_IMAGE_TAG>~${SERVER_IMAGE_TAG}~" k8s-deployment.yaml
# - sed -i "s~<RUN_CONFIG_SECRET_NAME>~${RUN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
# - sed -i "s~<GUNICORN_CONFIG_SECRET_NAME>~${GUNICORN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
# - cat k8s-deployment.yaml
# - kubectl apply -n ${CHEMBL_NS_STAGING} -f k8s-deployment.yaml
# - kubectl get pods -n ${CHEMBL_NS_STAGING}
# - cat ${INGRESS_STAGING_FILE_PATH}
# - kubectl apply -n ${CHEMBL_NS_STAGING} -f ${INGRESS_STAGING_FILE_PATH}
# - kubectl describe ingress -n ${CHEMBL_NS_STAGING}
# - cat ${AUTOSCALER_STAGING_FILE_PATH}
# - kubectl apply -n ${CHEMBL_NS_STAGING} -f ${AUTOSCALER_STAGING_FILE_PATH}
##
### ----------------------------------------------------------------------------------------------------------------------
### Test staging
### ----------------------------------------------------------------------------------------------------------------------
#functional_tests:
# stage: test_staging
# image:
# name: $SERVER_IMAGE_TAG
# entrypoint: [""]
# only:
# - staging
# except:
# - schedules
# environment:
# name: staging
# url: ${SERVER_URL_STAGING}
# script:
# - set -x
# - functional_tests/run_functional_tests.py ${SERVER_URL_STAGING} https://wwwdev.ebi.ac.uk/chembl/interface_api/delayed_jobs
#
### ----------------------------------------------------------------------------------------------------------------------
### Deployment to prod hx
### ----------------------------------------------------------------------------------------------------------------------
#prepare_config_prod_hx:
# stage: prepare_config_hx
# image:
# name: alpine/git
# entrypoint: ["/bin/sh", "-c"]
# only:
# - master
# except:
# - schedules
# script:
# - rm -rf ${CONFIGS_FOLDER} || true
# - mkdir -p ${CONFIGS_FOLDER}
# - git clone https://${CONFIG_DEPLOY_GITLAB_USERNAME}:${CONFIG_DEPLOY_GITLAB_TOKEN}@${CONFIGURATIONS_REPO} ${CONFIGS_FOLDER}
# artifacts:
# paths:
# - ${CONFIG_PROD_HX_FILE_PATH}
# - ${CONFIG_GUNICORN_PROD_HX_FILE_PATH}
# - ${INGRESS_PROD_HX_FILE_PATH}
# - ${AUTOSCALER_PROD_HX_FILE_PATH}
## ----------------------------------------------------------------------------------------------------------------------
## Test staging
## ----------------------------------------------------------------------------------------------------------------------
functional_tests:
stage: test_staging
image:
name: $SERVER_IMAGE_TAG
entrypoint: [""]
only:
- staging
except:
- schedules
environment:
name: staging
url: ${SERVER_URL_STAGING}
script:
- set -x
- functional_tests/run_functional_tests.py ${SERVER_URL_STAGING}
## ----------------------------------------------------------------------------------------------------------------------
## Deployment to prod hx
## ----------------------------------------------------------------------------------------------------------------------
prepare_config_prod_hx:
stage: prepare_config_hx
image:
name: alpine/git
entrypoint: ["/bin/sh", "-c"]
only:
- master
except:
- schedules
script:
- rm -rf ${CONFIGS_FOLDER} || true
- mkdir -p ${CONFIGS_FOLDER}
- git clone https://${CONFIG_DEPLOY_GITLAB_USERNAME}:${CONFIG_DEPLOY_GITLAB_TOKEN}@${CONFIGURATIONS_REPO} ${CONFIGS_FOLDER}
artifacts:
paths:
- ${CONFIG_PROD_HX_FILE_PATH}
- ${CONFIG_GUNICORN_PROD_HX_FILE_PATH}
- ${AUTOSCALER_PROD_HX_FILE_PATH}
#
#deploy_to_prod_hx:
# stage: deploy_to_prod_hx
......
......@@ -39,4 +39,4 @@ services:
target: task-status-daemon
args:
<<: *build_args
image: "${STATUS_DAEMON_IMAGE_TAG:-chembl/es-subset-generator-tasks-daemon:prod}"
\ No newline at end of file
image: "${TASKS_DAEMON_IMAGE_TAG:-chembl/es-subset-generator-tasks-daemon:prod}"
\ No newline at end of file
#!/usr/bin/env python3
# pylint: disable=import-error
"""
Script that runs the functional tests for the app
"""
import argparse
PARSER = argparse.ArgumentParser()
PARSER.add_argument('server_base_path', help='server base path to run the tests against',
default='http://127.0.0.1:5000', nargs='?')
ARGS = PARSER.parse_args()
def run():
"""
Runs all functional tests
"""
print(f'Running functional tests on {ARGS.server_base_path}')
if __name__ == "__main__":
run()
apiVersion: apps/v1
kind: Deployment
metadata:
name: es-subset-generator
namespace: <NAMESPACE>
spec:
replicas: <NUM_REPLICAS>
selector:
matchLabels:
app: es-subset-generator
template:
metadata:
labels:
app: es-subset-generator
spec:
containers:
- name: es-subset-generator-api
image: <SERVER_IMAGE_TAG>
imagePullPolicy: Always
resources:
limits:
cpu: <CPU_LIMIT>
requests:
cpu: <CPU_REQUESTED>
volumeMounts:
- name: run-config
mountPath: '/etc/run_config/'
readOnly: true
- name: gunicorn-config
mountPath: '/etc/gunicorn_config/'
readOnly: true
env:
- name: HTTP_PROXY
value: "<HTTP_PROXY>"
- name: http_proxy
value: "<HTTP_PROXY>"
- name: HTTPS_PROXY
value: "<HTTPS_PROXY>"
- name: https_proxy
value: "<HTTPS_PROXY>"
- name: NO_PROXY
value: "<NO_PROXY>"
- name: no_proxy
value: "<NO_PROXY>"
- name: es-subset-generator-tasks-daemon
image: <TASKS_DAEMON_IMAGE_TAG>
imagePullPolicy: Always
resources:
limits:
cpu: <CPU_LIMIT>
requests:
cpu: <CPU_REQUESTED>
volumeMounts:
- name: run-config
mountPath: '/etc/run_config/'
readOnly: true
- name: gunicorn-config
mountPath: '/etc/gunicorn_config/'
readOnly: true
env:
- name: HTTP_PROXY
value: "<HTTP_PROXY>"
- name: http_proxy
value: "<HTTP_PROXY>"
- name: HTTPS_PROXY
value: "<HTTPS_PROXY>"
- name: https_proxy
value: "<HTTPS_PROXY>"
- name: NO_PROXY
value: "<NO_PROXY>"
- name: no_proxy
value: "<NO_PROXY>"
volumes:
- name: run-config
secret:
secretName: <RUN_CONFIG_SECRET_NAME>
- name: gunicorn-config
secret:
secretName: <GUNICORN_CONFIG_SECRET_NAME>
---
apiVersion: v1
kind: Service
metadata:
name: es-subset-generator
namespace: <NAMESPACE>
spec:
type: NodePort
selector:
app: es-subset-generator
ports:
- port: 5000
targetPort: 5000
"""
WSGI config for the ES Subset Generator API Server.
"""
from app import create_app
FLASK_APP = create_app()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment