.gitlab-ci.yml 11.5 KB
Newer Older
1 2 3 4 5 6
# CI/CD Pipeline
# dev branch -> staging
# dev branch -> internal (Custom Downloads + Entity Viewer)
# master branch -> live

# Define stages for the CI/CD pipeline
7 8 9 10 11
stages:
  - test
  - test_build_static
  - build_docker_images
  - deploy
Andrey Azov's avatar
Andrey Azov committed
12

13
variables:
14 15
  CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}
  KUBE_CONTEXT: ens-dev-ctx
16
  DEPLOYENV: dev
17
  ENVIRONMENT: production
18
  DOCKER_TLS_CERTDIR: ""
Andrey Azov's avatar
Andrey Azov committed
19

20 21 22
# Template to build static assets
.build-static:
  stage: test_build_static
Andrey Azov's avatar
Andrey Azov committed
23
  image: node:14.16.0
24 25 26

  before_script:
  - cd src/ensembl
Andrey Azov's avatar
Andrey Azov committed
27
  - npm install --global npm@7.7.4
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
  - npm ci

  script:
  - npm run test
  - npm run build

  artifacts:
    name: static_assets
    paths:
    - src/ensembl/dist/

# Template to build nginx docker image for WP k8s cluster
# WP k8s cluster uses ingress
.build-nginx-wp:
  image: docker

  services:
    - docker:dind

  stage: build_docker_images

  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  script:
    - apk update && apk add git
    - git clone --depth 1 https://github.com/Ensembl/ensembl-2020-static-assests.git
    - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-nginx.git
    - git -C ensembl-client-nginx checkout migration/wp-hx/ingress
    - docker build -t ${CONTAINER_IMAGE} -f ensembl-client-nginx/Dockerfile --no-cache .
    - docker push ${CONTAINER_IMAGE}
    - docker rmi ${CONTAINER_IMAGE}
    - docker logout $CI_REGISTRY

# Template to build nginx docker image for EHK k8s cluster
.build-nginx-ehk:
  image: docker

  services:
    - docker:dind

  stage: build_docker_images

  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  script:
  - apk update && apk add git
  - git clone --depth 1 https://github.com/Ensembl/ensembl-2020-static-assests.git
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-nginx.git
  - sed -i "s/<DEPLOYMENT_ENV>/${DEPLOYENV}/g" ensembl-client-nginx/config/conf.d/local.conf
  - docker build -t ${CONTAINER_IMAGE} -f ensembl-client-nginx/Dockerfile --no-cache .
  - docker push ${CONTAINER_IMAGE}
  - docker rmi ${CONTAINER_IMAGE}
  - docker logout "$CI_REGISTRY"

84 85
  needs:
    - Test_N_Build
86 87 88 89

# Template to deploy to WP k8s cluster
.deploy-wp:
  stage: deploy
90
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
91 92 93 94 95 96
  script:
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-caas-deploy.git
  - git -C ensembl-client-caas-deploy/ checkout migration/wp-hx
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_deployment.yaml

97 98 99
# Template to deploy to WP k8s cluster
.deploy-wp-feature:
  stage: deploy
100
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
101 102 103
  script:
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-caas-deploy.git
  - git -C ensembl-client-caas-deploy/ checkout deployfeature
104
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml 
105
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
106
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-client-caas-deploy/ensembl_client_service_node.yaml
107 108 109
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_service_node.yaml

110 111 112
# Template to deploy review-app to WP k8s cluster
.deploy-wp-review:
  stage: deploy
113
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
114 115 116 117 118 119 120 121 122 123
  script:
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-caas-deploy.git
  - git -C ensembl-client-caas-deploy/ checkout wp-review-apps
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml 
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_deployment.yaml

# Template for stopping review app - Do cleanup here 
.stop-wp-review:
  stage: deploy
124
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
125 126 127 128 129 130 131 132
  variables:
    GIT_STRATEGY: none
  when: manual
  script:
  - kubectl delete ingress ensembl-client-${CI_COMMIT_REF_SLUG}-ingress
  - kubectl delete svc ensembl-client-${CI_COMMIT_REF_SLUG}-svc
  - kubectl delete deploy ensembl-client-${CI_COMMIT_REF_SLUG}-deployment

133 134 135
# Template to deploy to EHK k8s cluster
.deploy-ehk:
  stage: deploy
136
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
137 138 139 140 141 142 143 144 145 146 147 148 149
  before_script:
  - export KUBECONFIG=/etc/deploy/config
  - mkdir -p /etc/deploy
  - echo ${EMBASSY_KUBECONFIG} | base64 -d > ${KUBECONFIG}
  script:
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-caas-deploy.git
  - git -C ensembl-client-caas-deploy checkout refactor-manifest
  - sed -i "s/<VERSION>/${CI_COMMIT_SHORT_SHA}-${DEPLOYENV}/g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - sed -i "s/<DEPLOYMNET_ENV>/${DEPLOYENV}/g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - cat ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - kubectl config use-context ${KUBE_CONTEXT}
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_deployment.yaml

150
Test:
Andrey Azov's avatar
Andrey Azov committed
151
  image: node:14.16.0
152
  stage: test
Andrey Azov's avatar
Andrey Azov committed
153

154 155 156 157 158
  before_script:
  - cd src/ensembl
  - npm ci

  script:
159
  - npm run check-types
160 161
  - npm run test

162 163
  only:
  - /^nodeploy\/.*$/
164 165 166 167
  except:
  - dev
  - master

168 169 170
# Job to build static asset for staging and live environment
# dev branch -> Staging
# master branch -> Live
171
Test_N_Build:
172
  extends: .build-static
173

174 175 176 177
  variables:
    GOOGLE_ANALYTICS_KEY: ${GA_KEY}
    ENVIRONMENT: production
    API_HOST: ""
178 179 180 181 182 183 184 185 186

  script:
  - npm run test
  - npm run build

  only:
  - dev
  - master

187 188
# Job to build static asset for internal environment
# dev branch -> Internal
189
Test_N_Build:internal:
190
  extends: .build-static
191

192 193 194 195
  variables:
    GOOGLE_ANALYTICS_KEY: ${GA_KEY}
    ENVIRONMENT: internal
    API_HOST: ""
196 197

  artifacts:
198
    name: static_assets
199 200 201 202
    paths:
    - src/ensembl/dist/

  only:
203
  - dev
204

205
Test_N_Build:review:
206 207 208 209
  extends: .build-static
  variables:
    ENVIRONMENT: development
    API_HOST: ""
210

211 212 213 214
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
215

216 217 218 219 220 221 222 223
# Job to build nginx docker image for staging environment (EHK k8s cluster)
Nginx:Staging-EHK:
  extends: .build-nginx-ehk
  variables:
    DEPLOYENV: staging
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-staging
  only:
  - dev
224

225 226 227 228 229 230 231 232
# Job to build nginx docker image for live environment (EHK k8s cluster)
Nginx:Live-EHK:
  extends: .build-nginx-ehk
  variables:
    DEPLOYENV: prod
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod
  only:
  - master
233

234 235 236 237 238 239
# Job to build nginx docker image for staging environment
# dev branch -> Staging
Nginx:Staging-WP:
  extends: .build-nginx-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-stage-ing
240 241
  only:
  - dev
242 243
  needs:
    - Test_N_Build
244 245 246 247 248 249 250 251

# Job to build nginx docker image for live environment
# master branch -> Live
Nginx:Live-WP:
  extends: .build-nginx-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod-ing
  only:
252
  - master
253 254
  needs:
    - Test_N_Build
255 256 257 258 259 260 261 262

# Job to build nginx docker image for staging and internal environment
# dev branch -> internal
Nginx:Internal-EHK:
  extends: .build-nginx-ehk
  variables:
    DEPLOYENV: internal
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal
263
  only:
264
  - dev
265 266
  needs:
    - Test_N_Build:internal
267

Jyothish's avatar
Jyothish committed
268 269 270 271 272 273 274 275 276
# Job to build nginx docker image for staging and internal environment at WP
# dev branch -> internal
Nginx:Internal-WP:
  extends: .build-nginx-wp
  variables:
    DEPLOYENV: internal
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal-ing
  only:
  - dev
277 278
  needs:
    - Test_N_Build:internal
Jyothish's avatar
Jyothish committed
279

280
Nginx:review:
281
  extends: .build-nginx-wp
282 283 284
  variables:
    DEPLOYENV: dev
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
285 286 287 288
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
289

290
  needs:
291
    - Test_N_Build:review
292

293 294 295 296 297 298 299
# Job to deploy to staging environment (EHK k8s cluster)
Staging:EHK-HH:
  extends: .deploy-ehk
  variables:
    DEPLOYENV: staging
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-staging
    KUBE_CONTEXT: ens-stage-ctx
300
  only:
301
  - dev
302 303
  needs:
    - Nginx:Staging-EHK
304 305 306 307 308 309 310 311 312 313

# Job to deploy to staging environment (WP-HX k8s cluster)
Staging:WP-HX:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-stage-ing
  environment:
    name : wp-hx-staging
  only:
  - dev
314 315 316
  needs:
    - Test_N_Build
    - Nginx:Staging-WP
317

318 319 320 321 322 323 324 325 326
# Job to deploy to staging environment (WP-HH k8s cluster)
Staging:WP-HH:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-stage-ing
  environment:
    name : wp-hh-staging
  only:
  - dev
327 328 329
  needs:
    - Test_N_Build
    - Nginx:Staging-WP
330

331 332 333 334 335 336 337
# Job to deploy to live environment (EHK k8s cluster)
Live:EHK-HH:
  extends: .deploy-ehk
  variables:
    DEPLOYENV: prod
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod
    KUBE_CONTEXT: ens-prod-ctx
338
  only:
339
  - master
340 341
  needs:
    - Nginx:Live-EHK
342

Kamal Dodiya's avatar
Kamal Dodiya committed
343 344 345 346 347 348 349 350 351
# Job to deploy to live environment (WP-HX k8s cluster)
Live:WP-HX:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod-ing
  environment:
    name : wp-hx-live
  only:
  - master
352 353 354
  needs:
    - Test_N_Build
    - Nginx:Live-WP
Kamal Dodiya's avatar
Kamal Dodiya committed
355 356 357 358 359 360 361 362 363 364

# Job to deploy to live environment (WP-HH k8s cluster)
Live:WP-HH:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod-ing
  environment:
    name : wp-hh-live
  only:
  - master
365 366 367
  needs:
    - Test_N_Build
    - Nginx:Live-WP
Kamal Dodiya's avatar
Kamal Dodiya committed
368

369 370 371 372 373 374 375
# Job to deploy to internal environment (EHK k8s cluster)
Internal:EHK-HH:
  extends: .deploy-ehk
  variables:
    DEPLOYENV: internal
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal
    KUBE_CONTEXT: ens-dev-ctx
376
  only:
377
  - dev
378 379 380
  needs:
    - Test_N_Build:internal
    - Nginx:Internal-EHK
381

Jyothish's avatar
Jyothish committed
382 383 384 385 386 387 388 389 390
# Job to deploy to internal environment (WP-HX k8s cluster)
Internal:WP-HX:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal-ing
  environment:
    name : wp-hx-internal
  only:
  - dev
391 392 393
  needs:
    - Test_N_Build:internal
    - Nginx:Internal-WP
Jyothish's avatar
Jyothish committed
394

395 396 397 398 399 400 401 402 403
# Job to deploy to internal environment (WP-HH k8s cluster)
Internal:WP-HH:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal-ing
  environment:
    name : wp-hh-internal
  only:
  - dev
404 405 406
  needs:
    - Test_N_Build:internal
    - Nginx:Internal-WP
407

408 409 410 411 412 413
Feature:EHK-HH:
  extends: .deploy-ehk
  variables:
    DEPLOYENV: ${CI_COMMIT_REF_SLUG}
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
    KUBE_CONTEXT: ens-dev-ctx
414

415 416 417 418
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
419

420
  needs:
421 422
    - Test_N_Build:review
    - Nginx:review
423

424 425
Review:WP-HX:
  extends: .deploy-wp-review
426 427 428
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
  environment:
429 430 431 432 433 434 435 436 437
    name: review/$CI_COMMIT_REF_SLUG
    url: http://$CI_COMMIT_REF_SLUG.review.ensembl.org
    on_stop: stop_review
    kubernetes:
      namespace: ensembl-dev
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
438
  needs:
439 440 441 442 443 444 445 446 447 448 449 450 451
    - Test_N_Build:review
    - Nginx:review

stop_review:
  extends: .stop-wp-review
  environment:
    name: review/$CI_COMMIT_REF_SLUG
    action: stop
    kubernetes:
      namespace: ensembl-dev
  except:
  - dev
  - master
Andrey Azov's avatar
Andrey Azov committed
452
  - /^nodeploy\/.*$/