.gitlab-ci.yml 11.5 KB
Newer Older
1
2
# CI/CD Pipeline
# dev branch -> staging
3
# master branch -> internal (Custom Downloads)
4
5
6
# master branch -> live

# Define stages for the CI/CD pipeline
7
8
9
10
11
stages:
  - test
  - test_build_static
  - build_docker_images
  - deploy
Andrey Azov's avatar
Andrey Azov committed
12

13
variables:
14
15
  CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}
  KUBE_CONTEXT: ens-dev-ctx
16
  DEPLOYENV: dev
17
  ENVIRONMENT: production
18
  DOCKER_TLS_CERTDIR: ""
Andrey Azov's avatar
Andrey Azov committed
19

20
21
22
# Template to build static assets
.build-static:
  stage: test_build_static
Andrey Azov's avatar
Andrey Azov committed
23
  image: node:14.16.0
24
25
26

  before_script:
  - cd src/ensembl
Andrey Azov's avatar
Andrey Azov committed
27
  - npm install --global npm@7.7.4
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
  - npm ci

  script:
  - npm run test
  - npm run build

  artifacts:
    name: static_assets
    paths:
    - src/ensembl/dist/

# Template to build nginx docker image for WP k8s cluster
# WP k8s cluster uses ingress
.build-nginx-wp:
  image: docker

  services:
    - docker:dind

  stage: build_docker_images

  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  script:
    - apk update && apk add git
    - git clone --depth 1 https://github.com/Ensembl/ensembl-2020-static-assests.git
    - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-nginx.git
    - git -C ensembl-client-nginx checkout migration/wp-hx/ingress
    - docker build -t ${CONTAINER_IMAGE} -f ensembl-client-nginx/Dockerfile --no-cache .
    - docker push ${CONTAINER_IMAGE}
    - docker rmi ${CONTAINER_IMAGE}
    - docker logout $CI_REGISTRY

# Template to build nginx docker image for EHK k8s cluster
.build-nginx-ehk:
  image: docker

  services:
    - docker:dind

  stage: build_docker_images

  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  script:
  - apk update && apk add git
  - git clone --depth 1 https://github.com/Ensembl/ensembl-2020-static-assests.git
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-nginx.git
  - sed -i "s/<DEPLOYMENT_ENV>/${DEPLOYENV}/g" ensembl-client-nginx/config/conf.d/local.conf
  - docker build -t ${CONTAINER_IMAGE} -f ensembl-client-nginx/Dockerfile --no-cache .
  - docker push ${CONTAINER_IMAGE}
  - docker rmi ${CONTAINER_IMAGE}
  - docker logout "$CI_REGISTRY"

84
85
  needs:
    - Test_N_Build
86
87
88
89

# Template to deploy to WP k8s cluster
.deploy-wp:
  stage: deploy
90
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
91
92
93
94
95
96
  script:
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-caas-deploy.git
  - git -C ensembl-client-caas-deploy/ checkout migration/wp-hx
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_deployment.yaml

97
98
99
# Template to deploy to WP k8s cluster
.deploy-wp-feature:
  stage: deploy
100
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
101
102
103
  script:
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-caas-deploy.git
  - git -C ensembl-client-caas-deploy/ checkout deployfeature
104
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml 
105
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
106
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-client-caas-deploy/ensembl_client_service_node.yaml
107
108
109
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_service_node.yaml

110
111
112
# Template to deploy review-app to WP k8s cluster
.deploy-wp-review:
  stage: deploy
113
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
114
115
116
117
118
119
120
121
122
123
  script:
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-caas-deploy.git
  - git -C ensembl-client-caas-deploy/ checkout wp-review-apps
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml 
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_deployment.yaml

# Template for stopping review app - Do cleanup here 
.stop-wp-review:
  stage: deploy
124
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
125
126
127
128
129
130
131
132
  variables:
    GIT_STRATEGY: none
  when: manual
  script:
  - kubectl delete ingress ensembl-client-${CI_COMMIT_REF_SLUG}-ingress
  - kubectl delete svc ensembl-client-${CI_COMMIT_REF_SLUG}-svc
  - kubectl delete deploy ensembl-client-${CI_COMMIT_REF_SLUG}-deployment

133
134
135
# Template to deploy to EHK k8s cluster
.deploy-ehk:
  stage: deploy
136
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
137
138
139
140
141
142
143
144
145
146
147
148
149
  before_script:
  - export KUBECONFIG=/etc/deploy/config
  - mkdir -p /etc/deploy
  - echo ${EMBASSY_KUBECONFIG} | base64 -d > ${KUBECONFIG}
  script:
  - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-caas-deploy.git
  - git -C ensembl-client-caas-deploy checkout refactor-manifest
  - sed -i "s/<VERSION>/${CI_COMMIT_SHORT_SHA}-${DEPLOYENV}/g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - sed -i "s/<DEPLOYMNET_ENV>/${DEPLOYENV}/g" ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - cat ensembl-client-caas-deploy/ensembl_client_deployment.yaml
  - kubectl config use-context ${KUBE_CONTEXT}
  - kubectl apply -f ensembl-client-caas-deploy/ensembl_client_deployment.yaml

150
Test:
Andrey Azov's avatar
Andrey Azov committed
151
  image: node:14.16.0
152
  stage: test
Andrey Azov's avatar
Andrey Azov committed
153

154
155
156
157
158
  before_script:
  - cd src/ensembl
  - npm ci

  script:
159
  - npm run check-types
160
161
  - npm run test

162
163
  only:
  - /^nodeploy\/.*$/
164
165
166
167
  except:
  - dev
  - master

168
169
170
# Job to build static asset for staging and live environment
# dev branch -> Staging
# master branch -> Live
171
Test_N_Build:
172
  extends: .build-static
173

174
175
176
177
  variables:
    GOOGLE_ANALYTICS_KEY: ${GA_KEY}
    ENVIRONMENT: production
    API_HOST: ""
178
179
180
181
182
183
184
185
186

  script:
  - npm run test
  - npm run build

  only:
  - dev
  - master

187
# Job to build static asset for internal environment
188
# master branch -> Internal
189
Test_N_Build:internal:
190
  extends: .build-static
191

192
193
194
195
  variables:
    GOOGLE_ANALYTICS_KEY: ${GA_KEY}
    ENVIRONMENT: internal
    API_HOST: ""
196
197

  artifacts:
198
    name: static_assets
199
200
201
202
    paths:
    - src/ensembl/dist/

  only:
203
  - master
204

205
Test_N_Build:review:
206
207
208
209
  extends: .build-static
  variables:
    ENVIRONMENT: development
    API_HOST: ""
210

211
212
213
214
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
215

216
217
218
219
220
221
222
223
# Job to build nginx docker image for staging environment (EHK k8s cluster)
Nginx:Staging-EHK:
  extends: .build-nginx-ehk
  variables:
    DEPLOYENV: staging
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-staging
  only:
  - dev
224

225
226
227
228
229
230
231
232
# Job to build nginx docker image for live environment (EHK k8s cluster)
Nginx:Live-EHK:
  extends: .build-nginx-ehk
  variables:
    DEPLOYENV: prod
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod
  only:
  - master
233

234
235
236
237
238
239
# Job to build nginx docker image for staging environment
# dev branch -> Staging
Nginx:Staging-WP:
  extends: .build-nginx-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-stage-ing
240
241
  only:
  - dev
242
243
  needs:
    - Test_N_Build
244
245
246
247
248
249
250
251

# Job to build nginx docker image for live environment
# master branch -> Live
Nginx:Live-WP:
  extends: .build-nginx-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod-ing
  only:
252
  - master
253
254
  needs:
    - Test_N_Build
255
256

# Job to build nginx docker image for staging and internal environment
257
# master branch -> internal
258
259
260
261
262
Nginx:Internal-EHK:
  extends: .build-nginx-ehk
  variables:
    DEPLOYENV: internal
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal
263
  only:
264
  - master
265
266
  needs:
    - Test_N_Build:internal
267

Jyothish's avatar
Jyothish committed
268
# Job to build nginx docker image for staging and internal environment at WP
269
# master branch -> internal
Jyothish's avatar
Jyothish committed
270
271
272
273
274
275
Nginx:Internal-WP:
  extends: .build-nginx-wp
  variables:
    DEPLOYENV: internal
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal-ing
  only:
276
  - master
277
278
  needs:
    - Test_N_Build:internal
Jyothish's avatar
Jyothish committed
279

280
Nginx:review:
281
  extends: .build-nginx-wp
282
283
284
  variables:
    DEPLOYENV: dev
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
285
286
287
288
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
289

290
  needs:
291
    - Test_N_Build:review
292

293
294
295
296
297
298
299
# Job to deploy to staging environment (EHK k8s cluster)
Staging:EHK-HH:
  extends: .deploy-ehk
  variables:
    DEPLOYENV: staging
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-staging
    KUBE_CONTEXT: ens-stage-ctx
300
  only:
301
  - dev
302
303
  needs:
    - Nginx:Staging-EHK
304
305
306
307
308
309
310
311
312
313

# Job to deploy to staging environment (WP-HX k8s cluster)
Staging:WP-HX:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-stage-ing
  environment:
    name : wp-hx-staging
  only:
  - dev
314
315
316
  needs:
    - Test_N_Build
    - Nginx:Staging-WP
317

318
319
320
321
322
323
324
325
326
# Job to deploy to staging environment (WP-HH k8s cluster)
Staging:WP-HH:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-stage-ing
  environment:
    name : wp-hh-staging
  only:
  - dev
327
328
329
  needs:
    - Test_N_Build
    - Nginx:Staging-WP
330

331
332
333
334
335
336
337
# Job to deploy to live environment (EHK k8s cluster)
Live:EHK-HH:
  extends: .deploy-ehk
  variables:
    DEPLOYENV: prod
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod
    KUBE_CONTEXT: ens-prod-ctx
338
  only:
339
  - master
340
341
  needs:
    - Nginx:Live-EHK
342

Kamal Dodiya's avatar
Kamal Dodiya committed
343
344
345
346
347
348
349
350
351
# Job to deploy to live environment (WP-HX k8s cluster)
Live:WP-HX:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod-ing
  environment:
    name : wp-hx-live
  only:
  - master
352
353
354
  needs:
    - Test_N_Build
    - Nginx:Live-WP
Kamal Dodiya's avatar
Kamal Dodiya committed
355
356
357
358
359
360
361
362
363
364

# Job to deploy to live environment (WP-HH k8s cluster)
Live:WP-HH:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-prod-ing
  environment:
    name : wp-hh-live
  only:
  - master
365
366
367
  needs:
    - Test_N_Build
    - Nginx:Live-WP
Kamal Dodiya's avatar
Kamal Dodiya committed
368

369
370
371
372
373
374
375
# Job to deploy to internal environment (EHK k8s cluster)
Internal:EHK-HH:
  extends: .deploy-ehk
  variables:
    DEPLOYENV: internal
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal
    KUBE_CONTEXT: ens-dev-ctx
376
  only:
377
  - master
378
379
380
  needs:
    - Test_N_Build:internal
    - Nginx:Internal-EHK
381

Jyothish's avatar
Jyothish committed
382
383
384
385
386
387
388
389
# Job to deploy to internal environment (WP-HX k8s cluster)
Internal:WP-HX:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal-ing
  environment:
    name : wp-hx-internal
  only:
390
  - master
391
392
393
  needs:
    - Test_N_Build:internal
    - Nginx:Internal-WP
Jyothish's avatar
Jyothish committed
394

395
396
397
398
399
400
401
402
# Job to deploy to internal environment (WP-HH k8s cluster)
Internal:WP-HH:
  extends: .deploy-wp
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-internal-ing
  environment:
    name : wp-hh-internal
  only:
403
  - master
404
405
406
  needs:
    - Test_N_Build:internal
    - Nginx:Internal-WP
407

408
409
410
411
412
413
Feature:EHK-HH:
  extends: .deploy-ehk
  variables:
    DEPLOYENV: ${CI_COMMIT_REF_SLUG}
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
    KUBE_CONTEXT: ens-dev-ctx
414

415
416
417
418
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
419

420
  needs:
421
422
    - Test_N_Build:review
    - Nginx:review
423

424
425
Review:WP-HX:
  extends: .deploy-wp-review
426
427
428
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
  environment:
429
430
431
432
433
434
435
436
437
    name: review/$CI_COMMIT_REF_SLUG
    url: http://$CI_COMMIT_REF_SLUG.review.ensembl.org
    on_stop: stop_review
    kubernetes:
      namespace: ensembl-dev
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
438
  needs:
439
440
441
442
443
444
445
446
447
448
449
450
451
    - Test_N_Build:review
    - Nginx:review

stop_review:
  extends: .stop-wp-review
  environment:
    name: review/$CI_COMMIT_REF_SLUG
    action: stop
    kubernetes:
      namespace: ensembl-dev
  except:
  - dev
  - master
Andrey Azov's avatar
Andrey Azov committed
452
  - /^nodeploy\/.*$/