Commit 4a0e746f authored by David Mendez's avatar David Mendez
Browse files

Merge branch 'staging' into 'master'

Adapt Ci to new runners

See merge request !79
parents c16c2cbd d7712615
......@@ -4,6 +4,7 @@ variables:
REGISTRY_USER: chembl/chembl
SERVER_APPLICATION: delayed-jobs/delayed-jobs-api/delayed-jobs-server
SERVER_IMAGE_TAG: ${CI_REGISTRY}/${REGISTRY_USER}/${SERVER_APPLICATION}:${CI_COMMIT_SHORT_SHA}
SERVER_IMAGE_TAG_LATEST: ${CI_REGISTRY}/${REGISTRY_USER}/${SERVER_APPLICATION}:latest
DOCKER_DRIVER: overlay
CONFIGS_FOLDER: run_config
# Config unit tests
......@@ -43,7 +44,7 @@ stages:
build_docker_image_server:
image:
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker/compose:1.24.1
entrypoint: ["/bin/sh", "-c"]
entrypoint: [ "/bin/sh", "-c" ]
stage: build
variables:
DOCKER_HOST: tcp://docker:2375
......@@ -51,23 +52,18 @@ build_docker_image_server:
- schedules
services:
- docker:dind
tags:
- dind
before_script:
- apk add curl jq
- echo $DOCKERHUB_REGISTRY_PASSWORD | docker login -u $DOCKERHUB_REGISTRY_USER --password-stdin $DOCKERHUB_REGISTRY
- |
TOKEN=$(curl --user "${DOCKERHUB_REGISTRY_USER}:${DOCKERHUB_REGISTRY_PASSWORD}" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq --raw-output .token) && curl --head --header "Authorization: Bearer $TOKEN" "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest" 2>&1
- mkdir -p $HOME/.docker
- echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
script:
- set -x
# Check rate limit
- docker version # verify docker cli is there. Also prints server info
- echo ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}
- echo building $SERVER_IMAGE_TAG
- docker-compose build delayed-jobs-server-prod
- echo Deploying $SERVER_IMAGE_TAG
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- docker push $SERVER_IMAGE_TAG
- docker pull ${SERVER_IMAGE_TAG_LATEST} || true
- docker build --cache-from ${SERVER_IMAGE_TAG_LATEST} --tag ${SERVER_IMAGE_TAG} --tag ${SERVER_IMAGE_TAG_LATEST} --target production-server .
- docker push ${SERVER_IMAGE_TAG}
- docker push ${SERVER_IMAGE_TAG_LATEST}
# ----------------------------------------------------------------------------------------------------------------------
# QA
......@@ -76,13 +72,13 @@ pylint:
stage: test_and_qa
image:
name: $SERVER_IMAGE_TAG
entrypoint: [""]
entrypoint: [ "" ]
except:
- schedules
script:
- pwd
- ls
- echo 'disabled temporarily'
- pwd
- ls
- echo 'disabled temporarily'
# ----------------------------------------------------------------------------------------------------------------------
# Tests
......@@ -91,7 +87,7 @@ prepare_config_tests:
stage: prepare_config_tests
image:
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine/git
entrypoint: ["/bin/sh", "-c"]
entrypoint: [ "/bin/sh", "-c" ]
except:
- schedules
script:
......@@ -107,12 +103,12 @@ unit_tests:
stage: test_and_qa
image:
name: $SERVER_IMAGE_TAG
entrypoint: [""]
entrypoint: [ "" ]
except:
- schedules
script:
- env
- CONFIG_FILE_PATH=${CONFIG_UNIT_TESTS_FILE_PATH} python -m unittest
- env
- CONFIG_FILE_PATH=${CONFIG_UNIT_TESTS_FILE_PATH} python -m unittest
# ----------------------------------------------------------------------------------------------------------------------
# Deployment to staging
......@@ -121,7 +117,7 @@ prepare_config_staging:
stage: prepare_config_staging
image:
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine/git
entrypoint: ["/bin/sh", "-c"]
entrypoint: [ "/bin/sh", "-c" ]
only:
- staging
except:
......@@ -142,7 +138,7 @@ deploy_to_staging:
stage: deploy_to_staging
image:
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/lachlanevenson/k8s-kubectl:latest
entrypoint: ["/bin/sh", "-c"]
entrypoint: [ "/bin/sh", "-c" ]
only:
- staging
except:
......@@ -151,41 +147,41 @@ deploy_to_staging:
name: staging
url: ${SERVER_URL_STAGING}
script:
- set -x
- echo "$KUBE_CA_PEM_HX" > "$(pwd)/kube.ca.pem"
- kubectl config set-cluster ${KUBE_CLUS_NAME_HX} --server="${KUBE_URL_HX}" --certificate-authority="$(pwd)/kube.ca.pem"
- kubectl config set-credentials ${KUBE_USER_HX} --token="${KUBE_TOKEN_HX}"
- kubectl config set-context ${CHEMBL_NS_STAGING} --cluster=${KUBE_CLUS_NAME_HX} --user=${KUBE_USER_HX}
- kubectl config use-context ${CHEMBL_NS_STAGING}
- echo ${CHEMBL_NS_STAGING} ${KUBE_URL_HX} ${KUBE_CLUS_NAME_HX} ${KUBE_USER_HX}
- kubectl get pods -n ${CHEMBL_NS_STAGING}
- kubectl create secret generic ${ID_RSA_SECRET_NAME_STAGING} --from-file=ID_RSA_LSF=${ID_RSA_LSF_HX} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
- kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_STAGING} --from-file=RUN_CONFIG.yml=${CONFIG_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
- kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_STAGING} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
- cat ${CONFIG_STAGING_FILE_PATH}
- cat ${CONFIG_GUNICORN_STAGING_FILE_PATH}
- sed -i "s~<NAMESPACE>~${CHEMBL_NS_STAGING}~" k8s-deployment.yaml
- sed -i "s~<NUM_REPLICAS>~${NUM_REPLICAS_STAGING}~" k8s-deployment.yaml
- sed -i "s~<CPU_LIMIT>~${CPU_LIMIT_STAGING}~" k8s-deployment.yaml
- sed -i "s~<CPU_REQUESTED>~${CPU_REQUESTED_STAGING}~" k8s-deployment.yaml
- sed -i "s~<UID>~${CBL_ADM_UID}~" k8s-deployment.yaml
- sed -i "s~<GID>~${CBL_PUB_GID}~" k8s-deployment.yaml
- sed -i "s~<SERVER_IMAGE_TAG>~${SERVER_IMAGE_TAG}~" k8s-deployment.yaml
- sed -i "s~<NFS_MOUNT_PATH>~${NFS_MOUNT_PATH_STAGING}~" k8s-deployment.yaml
- sed -i "s~<NFS_FQDN>~${NFS_FQDN_HX}~" k8s-deployment.yaml
- sed -i "s~<NFS_PATH>~${NFS_PATH_HX}~" k8s-deployment.yaml
- sed -i "s~<ID_RSA_SECRET_NAME>~${ID_RSA_SECRET_NAME_STAGING}~" k8s-deployment.yaml
- sed -i "s~<RUN_CONFIG_SECRET_NAME>~${RUN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
- sed -i "s~<GUNICORN_CONFIG_SECRET_NAME>~${GUNICORN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
- sed -i "s~<NODE_PORT>~${DELAYED_JOBS_SERVICE_PORT_STAGING}~" k8s-deployment.yaml
- cat k8s-deployment.yaml
- kubectl apply -n ${CHEMBL_NS_STAGING} -f k8s-deployment.yaml
- kubectl get pods -n ${CHEMBL_NS_STAGING}
- cat ${INGRESS_STAGING_FILE_PATH}
- kubectl apply -n ${CHEMBL_NS_STAGING} -f ${INGRESS_STAGING_FILE_PATH}
- kubectl describe ingress -n ${CHEMBL_NS_STAGING}
- cat ${AUTOSCALER_STAGING_FILE_PATH}
- kubectl apply -n ${CHEMBL_NS_STAGING} -f ${AUTOSCALER_STAGING_FILE_PATH}
- set -x
- echo "$KUBE_CA_PEM_HX" > "$(pwd)/kube.ca.pem"
- kubectl config set-cluster ${KUBE_CLUS_NAME_HX} --server="${KUBE_URL_HX}" --certificate-authority="$(pwd)/kube.ca.pem"
- kubectl config set-credentials ${KUBE_USER_HX} --token="${KUBE_TOKEN_HX}"
- kubectl config set-context ${CHEMBL_NS_STAGING} --cluster=${KUBE_CLUS_NAME_HX} --user=${KUBE_USER_HX}
- kubectl config use-context ${CHEMBL_NS_STAGING}
- echo ${CHEMBL_NS_STAGING} ${KUBE_URL_HX} ${KUBE_CLUS_NAME_HX} ${KUBE_USER_HX}
- kubectl get pods -n ${CHEMBL_NS_STAGING}
- kubectl create secret generic ${ID_RSA_SECRET_NAME_STAGING} --from-file=ID_RSA_LSF=${ID_RSA_LSF_HX} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
- kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_STAGING} --from-file=RUN_CONFIG.yml=${CONFIG_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
- kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_STAGING} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
- cat ${CONFIG_STAGING_FILE_PATH}
- cat ${CONFIG_GUNICORN_STAGING_FILE_PATH}
- sed -i "s~<NAMESPACE>~${CHEMBL_NS_STAGING}~" k8s-deployment.yaml
- sed -i "s~<NUM_REPLICAS>~${NUM_REPLICAS_STAGING}~" k8s-deployment.yaml
- sed -i "s~<CPU_LIMIT>~${CPU_LIMIT_STAGING}~" k8s-deployment.yaml
- sed -i "s~<CPU_REQUESTED>~${CPU_REQUESTED_STAGING}~" k8s-deployment.yaml
- sed -i "s~<UID>~${CBL_ADM_UID}~" k8s-deployment.yaml
- sed -i "s~<GID>~${CBL_PUB_GID}~" k8s-deployment.yaml
- sed -i "s~<SERVER_IMAGE_TAG>~${SERVER_IMAGE_TAG}~" k8s-deployment.yaml
- sed -i "s~<NFS_MOUNT_PATH>~${NFS_MOUNT_PATH_STAGING}~" k8s-deployment.yaml
- sed -i "s~<NFS_FQDN>~${NFS_FQDN_HX}~" k8s-deployment.yaml
- sed -i "s~<NFS_PATH>~${NFS_PATH_HX}~" k8s-deployment.yaml
- sed -i "s~<ID_RSA_SECRET_NAME>~${ID_RSA_SECRET_NAME_STAGING}~" k8s-deployment.yaml
- sed -i "s~<RUN_CONFIG_SECRET_NAME>~${RUN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
- sed -i "s~<GUNICORN_CONFIG_SECRET_NAME>~${GUNICORN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
- sed -i "s~<NODE_PORT>~${DELAYED_JOBS_SERVICE_PORT_STAGING}~" k8s-deployment.yaml
- cat k8s-deployment.yaml
- kubectl apply -n ${CHEMBL_NS_STAGING} -f k8s-deployment.yaml
- kubectl get pods -n ${CHEMBL_NS_STAGING}
- cat ${INGRESS_STAGING_FILE_PATH}
- kubectl apply -n ${CHEMBL_NS_STAGING} -f ${INGRESS_STAGING_FILE_PATH}
- kubectl describe ingress -n ${CHEMBL_NS_STAGING}
- cat ${AUTOSCALER_STAGING_FILE_PATH}
- kubectl apply -n ${CHEMBL_NS_STAGING} -f ${AUTOSCALER_STAGING_FILE_PATH}
# ----------------------------------------------------------------------------------------------------------------------
# Test staging
......@@ -194,15 +190,15 @@ functional_tests_staging:
stage: test_staging
image:
name: $SERVER_IMAGE_TAG
entrypoint: [""]
entrypoint: [ "" ]
only:
- staging
- staging
environment:
name: staging
script:
- set -x
- sleep 30 # Give some time for the deployment to take effect.
- functional_tests/run_functional_tests.py ${DELAYED_JOBS_URL_STAGING} ${DJ_SERVER_ADMIN_USERNAME_STAGING} ${DJ_SERVER_ADMIN_PASSWORD_STAGING}
- set -x
- sleep 30 # Give some time for the deployment to take effect.
- functional_tests/run_functional_tests.py ${DELAYED_JOBS_URL_STAGING} ${DJ_SERVER_ADMIN_USERNAME_STAGING} ${DJ_SERVER_ADMIN_PASSWORD_STAGING}
# ----------------------------------------------------------------------------------------------------------------------
# Deployment to prod hx
......@@ -211,7 +207,7 @@ prepare_config_prod_hx:
stage: prepare_config_hx_hh
image:
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine/git
entrypoint: ["/bin/sh", "-c"]
entrypoint: [ "/bin/sh", "-c" ]
only:
- master
except:
......@@ -231,7 +227,7 @@ deploy_to_prod_hx:
stage: deploy_to_prod_hx_hh
image:
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/lachlanevenson/k8s-kubectl:latest
entrypoint: ["/bin/sh", "-c"]
entrypoint: [ "/bin/sh", "-c" ]
only:
- master
except:
......@@ -277,13 +273,13 @@ functional_tests_prod_hx:
stage: test_prod_hx_hh
image:
name: $SERVER_IMAGE_TAG
entrypoint: [""]
entrypoint: [ "" ]
only:
- master
- master
script:
- set -x
- sleep 30 # Give some time for the deployment to take effect.
- functional_tests/run_functional_tests.py ${DELAYED_JOBS_URL_PROD_HX} ${DJ_SERVER_ADMIN_USERNAME_PROD_HX} ${DJ_SERVER_ADMIN_PASSWORD_PROD_HX}
- set -x
- sleep 30 # Give some time for the deployment to take effect.
- functional_tests/run_functional_tests.py ${DELAYED_JOBS_URL_PROD_HX} ${DJ_SERVER_ADMIN_USERNAME_PROD_HX} ${DJ_SERVER_ADMIN_PASSWORD_PROD_HX}
# ----------------------------------------------------------------------------------------------------------------------
# Deployment to prod hh
......@@ -292,7 +288,7 @@ prepare_config_prod_hh:
stage: prepare_config_hx_hh
image:
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine/git
entrypoint: ["/bin/sh", "-c"]
entrypoint: [ "/bin/sh", "-c" ]
only:
- master
except:
......@@ -312,7 +308,7 @@ deploy_to_prod_hh:
stage: deploy_to_prod_hx_hh
image:
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/lachlanevenson/k8s-kubectl:latest
entrypoint: ["/bin/sh", "-c"]
entrypoint: [ "/bin/sh", "-c" ]
only:
- master
except:
......@@ -358,13 +354,13 @@ functional_tests_prod_hh:
stage: test_prod_hx_hh
image:
name: $SERVER_IMAGE_TAG
entrypoint: [""]
entrypoint: [ "" ]
only:
- master
- master
script:
- set -x
- sleep 30 # Give some time for the deployment to take effect.
- functional_tests/run_functional_tests.py ${DELAYED_JOBS_URL_PROD_HH} ${DJ_SERVER_ADMIN_USERNAME_PROD_HH} ${DJ_SERVER_ADMIN_PASSWORD_PROD_HH}
- set -x
- sleep 30 # Give some time for the deployment to take effect.
- functional_tests/run_functional_tests.py ${DELAYED_JOBS_URL_PROD_HH} ${DJ_SERVER_ADMIN_USERNAME_PROD_HH} ${DJ_SERVER_ADMIN_PASSWORD_PROD_HH}
# ----------------------------------------------------------------------------------------------------------------------
# Test prod public url
......@@ -373,10 +369,10 @@ functional_tests_prod_public:
stage: functional_tests_prod_public
image:
name: $SERVER_IMAGE_TAG
entrypoint: [""]
entrypoint: [ "" ]
only:
- master
needs: ["functional_tests_prod_hh", "functional_tests_prod_hx"]
needs: [ "functional_tests_prod_hh", "functional_tests_prod_hx" ]
script:
- set -x
- sleep 30 # Give some time for the deployment to take effect.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment