.gitlab-ci.yml 14.2 KB
Newer Older
David Mendez's avatar
David Mendez committed
1 2 3 4
variables:
  DOCKER_TLS_CERTDIR: ""
  GIT_STRATEGY: clone
  REGISTRY_USER: chembl/chembl
David Mendez's avatar
David Mendez committed
5 6
  SERVER_APPLICATION: main-web-interface/es-subset-generator/es-subset-generator-api
  TASKS_APPLICATION: main-web-interface/es-subset-generator/es-subset-generator-tasks-daemon
David Mendez's avatar
David Mendez committed
7
  SERVER_IMAGE_TAG: ${CI_REGISTRY}/${REGISTRY_USER}/${SERVER_APPLICATION}:${CI_COMMIT_SHORT_SHA}
8
  SERVER_IMAGE_TAG_LATEST: ${CI_REGISTRY}/${REGISTRY_USER}/${SERVER_APPLICATION}:latest
David Mendez's avatar
David Mendez committed
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
  DOCKER_DRIVER: overlay
  CONFIGS_FOLDER: run_config
  # Config unit tests
  CONFIG_UNIT_TESTS_FILE_PATH: ${CONFIGS_FOLDER}/unit_tests.yml
  # Config staging
  CONFIG_STAGING_FILE_PATH: ${CONFIGS_FOLDER}/staging.yml
  CONFIG_GUNICORN_STAGING_FILE_PATH: ${CONFIGS_FOLDER}/staging.gunicorn.config.py
  AUTOSCALER_STAGING_FILE_PATH: ${CONFIGS_FOLDER}/autoscaler-staging.yml
  # Config prod_hx
  CONFIG_PROD_HX_FILE_PATH: ${CONFIGS_FOLDER}/prod_hx.yml
  CONFIG_GUNICORN_PROD_HX_FILE_PATH: ${CONFIGS_FOLDER}/prod_hx.gunicorn.config.py
  AUTOSCALER_PROD_HX_FILE_PATH: ${CONFIGS_FOLDER}/autoscaler-prod_hx.yml
  # Config prod_hh
  CONFIG_PROD_HH_FILE_PATH: ${CONFIGS_FOLDER}/prod_hh.yml
  CONFIG_GUNICORN_PROD_HH_FILE_PATH: ${CONFIGS_FOLDER}/prod_hh.gunicorn.config.py
  AUTOSCALER_PROD_HH_FILE_PATH: ${CONFIGS_FOLDER}/autoscaler-prod_hh.yml

stages:
  - build
28
  - prepare_config_unit_tests
David Mendez's avatar
David Mendez committed
29
  - test_and_qa
30
  - prepare_config_deployment
31
  - deploy_to_staging
32
  - test_staging
33 34
  - deploy_to_prod_hx_hh
  - test_prod_hx_hh
35

David Mendez's avatar
David Mendez committed
36 37 38
# ----------------------------------------------------------------------------------------------------------------------
# build
# ----------------------------------------------------------------------------------------------------------------------
39
build_docker_image:
David Mendez's avatar
David Mendez committed
40
  image:
41
    name: docker/compose:1.24.1
David Mendez's avatar
David Mendez committed
42 43 44 45 46 47
    entrypoint: ["/bin/sh", "-c"]
  except:
    - schedules
  stage: build
  variables:
    DOCKER_HOST: tcp://docker:2375
David Mendez's avatar
David Mendez committed
48 49
  services:
    - docker:dind
50 51 52
  before_script:
    - mkdir -p $HOME/.docker
    - echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
David Mendez's avatar
David Mendez committed
53 54 55
  script:
    - set -x
    - docker version # verify docker cli is there. Also prints server info
56
    - echo building ${SERVER_IMAGE_TAG}
David Mendez's avatar
David Mendez committed
57
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
58 59 60 61
    - docker pull ${SERVER_IMAGE_TAG_LATEST} || true
    - docker build --cache-from ${SERVER_IMAGE_TAG_LATEST} --tag ${SERVER_IMAGE_TAG} --tag ${SERVER_IMAGE_TAG_LATEST} --target production-server .
    - docker push ${SERVER_IMAGE_TAG}
    - docker push ${SERVER_IMAGE_TAG_LATEST}
David Mendez's avatar
David Mendez committed
62 63 64 65 66 67 68 69 70 71 72 73 74

## ----------------------------------------------------------------------------------------------------------------------
## QA
## ----------------------------------------------------------------------------------------------------------------------
pylint:
  stage: test_and_qa
  image:
    name: $SERVER_IMAGE_TAG
    entrypoint: [""]
  except:
    - schedules
  script:
    - set -x
David Mendez's avatar
David Mendez committed
75
    - find . -iname "*.py" | xargs pylint
David Mendez's avatar
David Mendez committed
76

David Mendez's avatar
David Mendez committed
77 78 79
### ---------------------------------------------------------------------------------------------------------------------
### Tests
### ---------------------------------------------------------------------------------------------------------------------
80
prepare_config_unit_tests:
81
  stage: prepare_config_unit_tests
David Mendez's avatar
David Mendez committed
82
  image:
83
    name: alpine/git
David Mendez's avatar
David Mendez committed
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
    entrypoint: ["/bin/sh", "-c"]
  except:
    - schedules
  script:
    - set -x
    - mkdir -p ${CONFIGS_FOLDER}
    - git clone https://${CONFIG_DEPLOY_GITLAB_USERNAME}:${CONFIG_DEPLOY_GITLAB_TOKEN}@${CONFIGURATIONS_REPO} ${CONFIGS_FOLDER}
  artifacts:
    paths:
      - ${CONFIG_UNIT_TESTS_FILE_PATH}

unit_tests:
  stage: test_and_qa
  image:
    name: $SERVER_IMAGE_TAG
    entrypoint: [""]
  except:
    - schedules
  script:
    - set -x
104
    - CONFIG_FILE_PATH=${CONFIG_UNIT_TESTS_FILE_PATH} python -m unittest
105 106 107 108

## ----------------------------------------------------------------------------------------------------------------------
## Deployment to staging
## ----------------------------------------------------------------------------------------------------------------------
109
prepare_config_deployments:
110
  stage: prepare_config_deployment
111
  image:
112
    name: alpine/git
113 114 115
    entrypoint: ["/bin/sh", "-c"]
  only:
    - staging
116 117 118
    - master
  except:
    - schedules
119 120
  script:
    - set -x
121
    - rm -rf ${CONFIGS_FOLDER} || true
122 123 124 125 126 127 128
    - mkdir -p ${CONFIGS_FOLDER}
    - git clone https://${CONFIG_DEPLOY_GITLAB_USERNAME}:${CONFIG_DEPLOY_GITLAB_TOKEN}@${CONFIGURATIONS_REPO} ${CONFIGS_FOLDER}
  artifacts:
    paths:
      - ${CONFIG_STAGING_FILE_PATH}
      - ${CONFIG_GUNICORN_STAGING_FILE_PATH}
      - ${AUTOSCALER_STAGING_FILE_PATH}
129 130 131
      - ${CONFIG_PROD_HX_FILE_PATH}
      - ${CONFIG_GUNICORN_PROD_HX_FILE_PATH}
      - ${AUTOSCALER_PROD_HX_FILE_PATH}
132 133 134
      - ${CONFIG_PROD_HH_FILE_PATH}
      - ${CONFIG_GUNICORN_PROD_HH_FILE_PATH}
      - ${AUTOSCALER_PROD_HH_FILE_PATH}
135 136 137 138

deploy_to_staging:
  stage: deploy_to_staging
  image:
139
    name: lachlanevenson/k8s-kubectl:latest
140 141 142 143 144 145 146
    entrypoint: ["/bin/sh", "-c"]
  only:
    - staging
  except:
    - schedules
  script:
  - set -x
David Mendez's avatar
David Mendez committed
147 148 149 150
  - echo "$KUBE_CA_PEM_HH" > "$(pwd)/kube.ca.pem"
  - kubectl config set-cluster ${KUBE_CLUS_NAME_HH} --server="${KUBE_URL_HH}" --certificate-authority="$(pwd)/kube.ca.pem"
  - kubectl config set-credentials ${KUBE_USER_HH} --token="${KUBE_TOKEN_HH}"
  - kubectl config set-context ${CHEMBL_NS_STAGING} --cluster=${KUBE_CLUS_NAME_HH} --user=${KUBE_USER_HH}
151
  - kubectl config use-context ${CHEMBL_NS_STAGING}
David Mendez's avatar
David Mendez committed
152
  - echo ${CHEMBL_NS_STAGING} ${KUBE_URL_HH} ${KUBE_CLUS_NAME_HH} ${KUBE_USER_HH}
153
  - kubectl get pods -n ${CHEMBL_NS_STAGING}
David Mendez's avatar
David Mendez committed
154 155
  - kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_STAGING} --from-file=RUN_CONFIG.yml=${CONFIG_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run=client | kubectl apply -f -
  - kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_STAGING} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run=client | kubectl apply -f -
156 157
  - cat ${CONFIG_STAGING_FILE_PATH}
  - cat ${CONFIG_GUNICORN_STAGING_FILE_PATH}
158 159
  - sed -i "s~<NAMESPACE>~${CHEMBL_NS_STAGING}~" k8s-deployment.yaml
  - sed -i "s~<NUM_REPLICAS>~${NUM_REPLICAS_STAGING}~" k8s-deployment.yaml
David Mendez's avatar
David Mendez committed
160 161 162
  - sed -i "s~<HTTP_PROXY>~${HTTP_PROXY_HH}~g" k8s-deployment.yaml
  - sed -i "s~<HTTPS_PROXY>~${HTTPS_PROXY_HH}~g" k8s-deployment.yaml
  - sed -i "s~<NO_PROXY>~${NO_PROXY_HH_DEFAULT}~g" k8s-deployment.yaml
163 164
  - sed -i "s~<CPU_LIMIT>~${CPU_LIMIT_STAGING}~" k8s-deployment.yaml
  - sed -i "s~<CPU_REQUESTED>~${CPU_REQUESTED_STAGING}~" k8s-deployment.yaml
165 166
  - sed -i "s~<MEMORY_LIMIT>~${MEMORY_LIMIT_STAGING}~" k8s-deployment.yaml
  - sed -i "s~<MEMORY_REQUESTED>~${MEMORY_REQUESTED_STAGING}~" k8s-deployment.yaml
167 168 169
  - sed -i "s~<SERVER_IMAGE_TAG>~${SERVER_IMAGE_TAG}~" k8s-deployment.yaml
  - sed -i "s~<RUN_CONFIG_SECRET_NAME>~${RUN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
  - sed -i "s~<GUNICORN_CONFIG_SECRET_NAME>~${GUNICORN_CONFIG_SECRET_NAME_STAGING}~" k8s-deployment.yaml
170
  - sed -i "s~<NODE_PORT>~${ES_SUBSET_GENERATOR_NODE_PORT_STAGING}~" k8s-deployment.yaml
171
  - cat k8s-deployment.yaml
172 173 174 175
  - kubectl apply -n ${CHEMBL_NS_STAGING} -f k8s-deployment.yaml
  - kubectl get pods -n ${CHEMBL_NS_STAGING}
  - cat ${AUTOSCALER_STAGING_FILE_PATH}
  - kubectl apply -n ${CHEMBL_NS_STAGING} -f ${AUTOSCALER_STAGING_FILE_PATH}
176

177 178 179
## ----------------------------------------------------------------------------------------------------------------------
## Test staging
## ----------------------------------------------------------------------------------------------------------------------
180
functional_tests_staging:
181 182 183 184 185 186 187 188
  stage: test_staging
  image:
    name: $SERVER_IMAGE_TAG
    entrypoint: [""]
  only:
    - staging
  script:
  - set -x
189
  - sleep 30 # give some time for the pods to start
190
  - functional_tests/run_functional_tests.py ${ES_SUBSET_GENERATOR_URL_STAGING} ${ES_SUBSET_GENERATOR_STAGING_ADMIN_USERNAME} ${ES_SUBSET_GENERATOR_STAGING_ADMIN_PASSWORD} ${ES_PROXY_API_URL_STAGING}
191

192
## ---------------------------------------------------------------------------------------------------------------------
193
## Deployment to prod hx
194
## ---------------------------------------------------------------------------------------------------------------------
195
deploy_to_prod_hx:
196
  stage: deploy_to_prod_hx_hh
197
  image:
198
    name: lachlanevenson/k8s-kubectl:latest
199 200 201 202 203 204 205 206 207 208 209 210 211 212
    entrypoint: ["/bin/sh", "-c"]
  only:
    - master
  except:
    - schedules
  script:
    - set -x
    - echo "$KUBE_CA_PEM_HX" > "$(pwd)/kube.ca.pem"
    - kubectl config set-cluster ${KUBE_CLUS_NAME_HX} --server="${KUBE_URL_HX}" --certificate-authority="$(pwd)/kube.ca.pem"
    - kubectl config set-credentials ${KUBE_USER_HX} --token="${KUBE_TOKEN_HX}"
    - kubectl config set-context ${CHEMBL_NS_PROD} --cluster=${KUBE_CLUS_NAME_HX} --user=${KUBE_USER_HX}
    - kubectl config use-context ${CHEMBL_NS_PROD}
    - echo ${CHEMBL_NS_PROD} ${KUBE_URL_HX} ${KUBE_CLUS_NAME_HX} ${KUBE_USER_HX}
    - kubectl get pods -n ${CHEMBL_NS_PROD}
213 214
    - kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_PROD_HX} --from-file=RUN_CONFIG.yml=${CONFIG_PROD_HX_FILE_PATH} -o yaml -n ${CHEMBL_NS_PROD} --dry-run=client | kubectl apply -f -
    - kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_PROD_HX} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_PROD_HX_FILE_PATH} -o yaml -n ${CHEMBL_NS_PROD} --dry-run=client | kubectl apply -f -
215 216 217 218 219 220 221 222 223
    - cat ${CONFIG_PROD_HX_FILE_PATH}
    - cat ${CONFIG_GUNICORN_PROD_HX_FILE_PATH}
    - sed -i "s~<NAMESPACE>~${CHEMBL_NS_PROD}~" k8s-deployment.yaml
    - sed -i "s~<NUM_REPLICAS>~${NUM_REPLICAS_PROD_HX}~" k8s-deployment.yaml
    - sed -i "s~<HTTP_PROXY>~${HTTP_PROXY_HX}~g" k8s-deployment.yaml
    - sed -i "s~<HTTPS_PROXY>~${HTTPS_PROXY_HX}~g" k8s-deployment.yaml
    - sed -i "s~<NO_PROXY>~${NO_PROXY_HX_DEFAULT}~g" k8s-deployment.yaml
    - sed -i "s~<CPU_LIMIT>~${CPU_LIMIT_PROD_HX}~" k8s-deployment.yaml
    - sed -i "s~<CPU_REQUESTED>~${CPU_REQUESTED_PROD_HX}~" k8s-deployment.yaml
224 225
    - sed -i "s~<MEMORY_LIMIT>~${MEMORY_LIMIT_PROD_HX}~" k8s-deployment.yaml
    - sed -i "s~<MEMORY_REQUESTED>~${MEMORY_REQUESTED_PROD_HX}~" k8s-deployment.yaml
226 227 228
    - sed -i "s~<SERVER_IMAGE_TAG>~${SERVER_IMAGE_TAG}~" k8s-deployment.yaml
    - sed -i "s~<RUN_CONFIG_SECRET_NAME>~${RUN_CONFIG_SECRET_NAME_PROD_HX}~" k8s-deployment.yaml
    - sed -i "s~<GUNICORN_CONFIG_SECRET_NAME>~${GUNICORN_CONFIG_SECRET_NAME_PROD_HX}~" k8s-deployment.yaml
229
    - sed -i "s~<NODE_PORT>~${ES_SUBSET_GENERATOR_NODE_PORT_PROD_HX}~" k8s-deployment.yaml
230 231 232 233 234 235
    - cat k8s-deployment.yaml
    - kubectl apply -n ${CHEMBL_NS_PROD} -f k8s-deployment.yaml
    - kubectl get pods -n ${CHEMBL_NS_PROD}
    - cat ${AUTOSCALER_PROD_HX_FILE_PATH}
    - kubectl apply -n ${CHEMBL_NS_PROD} -f ${AUTOSCALER_PROD_HX_FILE_PATH}

236
functional_tests_prod_hx:
237
  stage: test_prod_hx_hh
238 239 240 241 242 243 244 245
  image:
    name: $SERVER_IMAGE_TAG
    entrypoint: [""]
  only:
    - master
  script:
  - set -x
  - sleep 30 # Give some time for the deployment to take effect.
246
  - functional_tests/run_functional_tests.py ${ES_SUBSET_GENERATOR_URL_PROD_HX} ${ES_SUBSET_GENERATOR_PROD_HX_ADMIN_USERNAME} ${ES_SUBSET_GENERATOR_PROD_HX_ADMIN_PASSWORD} ${ES_PROXY_API_URL_PROD_HX}
247

248 249 250 251
## ----------------------------------------------------------------------------------------------------------------------
## Deployment to prod hh
## ----------------------------------------------------------------------------------------------------------------------
deploy_to_prod_hh:
252
  stage: deploy_to_prod_hx_hh
253
  image:
254
    name: lachlanevenson/k8s-kubectl:latest
255 256 257 258 259 260 261 262 263 264 265 266 267 268
    entrypoint: ["/bin/sh", "-c"]
  only:
    - master
  except:
    - schedules
  script:
    - set -x
    - echo "$KUBE_CA_PEM_HH" > "$(pwd)/kube.ca.pem"
    - kubectl config set-cluster ${KUBE_CLUS_NAME_HH} --server="${KUBE_URL_HH}" --certificate-authority="$(pwd)/kube.ca.pem"
    - kubectl config set-credentials ${KUBE_USER_HH} --token="${KUBE_TOKEN_HH}"
    - kubectl config set-context ${CHEMBL_NS_PROD} --cluster=${KUBE_CLUS_NAME_HH} --user=${KUBE_USER_HH}
    - kubectl config use-context ${CHEMBL_NS_PROD}
    - echo ${CHEMBL_NS_PROD} ${KUBE_URL_HH} ${KUBE_CLUS_NAME_HH} ${KUBE_USER_HH}
    - kubectl get pods -n ${CHEMBL_NS_PROD}
David Mendez's avatar
David Mendez committed
269 270
    - kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_PROD_HH} --from-file=RUN_CONFIG.yml=${CONFIG_PROD_HH_FILE_PATH} -o yaml -n ${CHEMBL_NS_PROD} --dry-run=client | kubectl replace -f -
    - kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_PROD_HH} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_PROD_HH_FILE_PATH} -o yaml -n ${CHEMBL_NS_PROD} --dry-run=client | kubectl replace -f -
David Mendez's avatar
David Mendez committed
271 272 273 274 275 276 277 278 279 280
    - cat ${CONFIG_PROD_HH_FILE_PATH}
    - cat ${CONFIG_GUNICORN_PROD_HH_FILE_PATH}
    - sed -i "s~<NAMESPACE>~${CHEMBL_NS_PROD}~" k8s-deployment.yaml
    - sed -i "s~<NUM_REPLICAS>~${NUM_REPLICAS_PROD_HH}~" k8s-deployment.yaml
    - sed -i "s~<SERVER_IMAGE_TAG>~${SERVER_IMAGE_TAG}~" k8s-deployment.yaml
    - sed -i "s~<HTTP_PROXY>~${HTTP_PROXY_HH}~g" k8s-deployment.yaml
    - sed -i "s~<HTTPS_PROXY>~${HTTPS_PROXY_HH}~g" k8s-deployment.yaml
    - sed -i "s~<NO_PROXY>~${NO_PROXY_HH_DEFAULT}~g" k8s-deployment.yaml
    - sed -i "s~<CPU_LIMIT>~${CPU_LIMIT_PROD_HH}~" k8s-deployment.yaml
    - sed -i "s~<CPU_REQUESTED>~${CPU_REQUESTED_PROD_HH}~" k8s-deployment.yaml
281 282
    - sed -i "s~<MEMORY_LIMIT>~${MEMORY_LIMIT_PROD_HH}~" k8s-deployment.yaml
    - sed -i "s~<MEMORY_REQUESTED>~${MEMORY_REQUESTED_PROD_HH}~" k8s-deployment.yaml
David Mendez's avatar
David Mendez committed
283 284
    - sed -i "s~<RUN_CONFIG_SECRET_NAME>~${RUN_CONFIG_SECRET_NAME_PROD_HH}~" k8s-deployment.yaml
    - sed -i "s~<GUNICORN_CONFIG_SECRET_NAME>~${GUNICORN_CONFIG_SECRET_NAME_PROD_HH}~" k8s-deployment.yaml
David Mendez's avatar
David Mendez committed
285
    - sed -i "s~<NODE_PORT>~${ES_SUBSET_GENERATOR_NODE_PORT_PROD_HH}~" k8s-deployment.yaml
David Mendez's avatar
David Mendez committed
286 287 288 289 290
    - cat k8s-deployment.yaml
    - kubectl apply -n ${CHEMBL_NS_PROD} -f k8s-deployment.yaml
    - kubectl get pods -n ${CHEMBL_NS_PROD}
    - cat ${AUTOSCALER_PROD_HH_FILE_PATH}
    - kubectl apply -n ${CHEMBL_NS_PROD} -f ${AUTOSCALER_PROD_HH_FILE_PATH}
291 292

functional_tests_prod_hh:
293
  stage: test_prod_hx_hh
294 295 296 297 298 299 300 301
  image:
    name: $SERVER_IMAGE_TAG
    entrypoint: [""]
  only:
    - master
  script:
  - set -x
  - sleep 30 # Give some time for the deployment to take effect.
302
  - functional_tests/run_functional_tests.py ${ES_SUBSET_GENERATOR_URL_PROD_HH} ${ES_SUBSET_GENERATOR_PROD_HH_ADMIN_USERNAME} ${ES_SUBSET_GENERATOR_PROD_HH_ADMIN_PASSWORD} ${ES_PROXY_API_URL_PROD_HH}
303