.gitlab-ci.yml 9.85 KB
Newer Older
1 2
# CI/CD Pipeline
# dev branch -> staging
3
# master branch -> internal (Custom Downloads)
4 5
# master branch -> live

6 7 8
# include template to setup review app
include: '/gitlab-ci-templates/.setup-review-template.yaml'

9
# Define stages for the CI/CD pipeline
10
stages:
11
  - setup
12 13 14 15
  - test
  - test_build_static
  - build_docker_images
  - deploy
Andrey Azov's avatar
Andrey Azov committed
16

17
variables:
18 19
  CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}
  CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}
20
  KUBE_CONTEXT: ens-dev-ctx
21
  DEPLOYENV: dev
22
  ENVIRONMENT: production
23
  DOCKER_TLS_CERTDIR: ""
Andrey Azov's avatar
Andrey Azov committed
24

25 26 27
# Template to build static assets
.build-static:
  stage: test_build_static
28
  image: node:14.17.0
29 30 31

  before_script:
  - cd src/ensembl
32
  - npm install --global npm@7.12.1
33 34 35 36 37 38 39
  - npm ci

  script:
  - npm run test
  - npm run build

  artifacts:
40
    name: build_artifacts
41 42
    paths:
    - src/ensembl/dist/
43 44
    - src/ensembl/package.json
    - src/ensembl/package-lock.json
45 46 47

# Template to build nginx docker image for WP k8s cluster
# WP k8s cluster uses ingress
48
.build-nginx:
49 50 51 52 53 54 55 56 57 58 59 60 61 62
  image: docker

  services:
    - docker:dind

  stage: build_docker_images

  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  script:
    - apk update && apk add git
    - git clone --depth 1 https://github.com/Ensembl/ensembl-2020-static-assests.git
    - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-nginx.git
63
    - git -C ensembl-client-nginx checkout migration/ssr
64 65 66 67 68
    - docker build -t ${CONTAINER_IMAGE} -f ensembl-client-nginx/Dockerfile --no-cache .
    - docker push ${CONTAINER_IMAGE}
    - docker rmi ${CONTAINER_IMAGE}
    - docker logout $CI_REGISTRY

69 70 71
# Template to build node docker image for WP k8s cluster
# WP k8s cluster uses ingress
.build-node:
72 73 74 75 76 77 78 79 80 81 82
  image: docker

  services:
    - docker:dind

  stage: build_docker_images

  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  script:
83 84 85 86
    - docker build -t ${CONTAINER_NODE_IMAGE} -f src/ensembl/Dockerfile.node --no-cache .
    - docker push ${CONTAINER_NODE_IMAGE}
    - docker rmi ${CONTAINER_NODE_IMAGE}
    - docker logout $CI_REGISTRY
87

88
# Template to deploy to WP k8s cluster
89
.deploy:
90
  stage: deploy
91
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
92
  script:
93 94 95 96 97 98
  - git clone https://gitlab.ebi.ac.uk/ensembl-web/ensembl-k8s-manifests.git
  - git -C ensembl-k8s-manifests/ checkout wp-k8s
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_NODE_IMAGE}#g" ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
  - kubectl apply -f ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - kubectl apply -f ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
99

100
# Template to deploy review-app to WP k8s cluster
101
.deploy-review:
102
  stage: deploy
103
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
104
  script:
105 106 107 108 109 110 111 112
  - git clone https://gitlab.ebi.ac.uk/ensembl-web/ensembl-k8s-manifests.git
  - git -C ensembl-k8s-manifests/ checkout wp-k8s-review
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_NODE_IMAGE}#g" ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
  - kubectl apply -f ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - kubectl apply -f ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
113 114

# Template for stopping review app - Do cleanup here 
115
.stop-review:
116
  stage: deploy
117
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
118 119 120 121 122 123 124 125
  variables:
    GIT_STRATEGY: none
  when: manual
  script:
  - kubectl delete ingress ensembl-client-${CI_COMMIT_REF_SLUG}-ingress
  - kubectl delete svc ensembl-client-${CI_COMMIT_REF_SLUG}-svc
  - kubectl delete deploy ensembl-client-${CI_COMMIT_REF_SLUG}-deployment

126
Test:
Andrey Azov's avatar
Andrey Azov committed
127
  image: node:14.16.0
128
  stage: test
Andrey Azov's avatar
Andrey Azov committed
129

130 131 132 133 134
  before_script:
  - cd src/ensembl
  - npm ci

  script:
135
  - npm run check-types
136 137
  - npm run test

138 139
  only:
  - /^nodeploy\/.*$/
140 141 142 143
  except:
  - dev
  - master

144 145 146
# Job to build static asset for staging and live environment
# dev branch -> Staging
# master branch -> Live
147
Test_N_Build:
148
  extends: .build-static
149

150 151 152 153
  variables:
    GOOGLE_ANALYTICS_KEY: ${GA_KEY}
    ENVIRONMENT: production
    API_HOST: ""
154 155 156 157 158

  only:
  - dev
  - master

159
# Job to build static asset for internal environment
160
# master branch -> Internal
161
Test_N_Build:internal:
162
  extends: .build-static
163

164 165 166 167
  variables:
    GOOGLE_ANALYTICS_KEY: ${GA_KEY}
    ENVIRONMENT: internal
    API_HOST: ""
168 169

  only:
170
  - master
171

172
Test_N_Build:review:
173 174 175 176
  extends: .build-static
  variables:
    ENVIRONMENT: development
    API_HOST: ""
177

178 179 180 181
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
182

183 184 185 186
# Job to build nginx docker image for staging environment
# dev branch -> Staging
Nginx:Staging:
  extends: .build-nginx
187 188
  only:
  - dev
189 190
  needs:
    - Test_N_Build
191

192
# Job to build node docker image for staging environment
193
# dev branch -> Staging
194 195
Node:Staging:
  extends: .build-node
196 197
  only:
  - dev
198 199
  needs:
    - Test_N_Build
200 201 202

# Job to build nginx docker image for live environment
# master branch -> Live
203 204
Nginx:Live:
  extends: .build-nginx
205
  variables:
206
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-prod
207 208 209 210 211 212 213 214 215 216 217
  only:
  - master
  needs:
    - Test_N_Build

# Job to build node docker image for live environment
# master branch -> Live
Node:Live:
  extends: .build-node
  variables:
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-prod
218
  only:
219
  - master
220 221
  needs:
    - Test_N_Build
222

223
# Job to build nginx docker image for internal environment at WP
224
# master branch -> internal
225 226
Nginx:Internal:
  extends: .build-nginx
227 228
  variables:
    DEPLOYENV: internal
229
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-internal
230
  only:
231
  - master
232 233
  needs:
    - Test_N_Build:internal
234

235
# Job to build node docker image for internal environment at WP
236
# master branch -> internal
237 238
Node:Internal:
  extends: .build-node
Jyothish's avatar
Jyothish committed
239 240
  variables:
    DEPLOYENV: internal
241
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-internal
Jyothish's avatar
Jyothish committed
242
  only:
243
  - master
244 245
  needs:
    - Test_N_Build:internal
Jyothish's avatar
Jyothish committed
246

247 248
# Job to build nginx docker image for review environment at WP
# all other branch -> dev
249
Nginx:review:
250
  extends: .build-nginx
251 252
  variables:
    DEPLOYENV: dev
253
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
254 255 256 257
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
258

259
  needs:
260
    - Test_N_Build:review
261

262 263 264 265
# Job to build node docker image for review environment at WP
# all other branch -> dev
Node:review:
  extends: .build-node
266
  variables:
267 268 269
    DEPLOYENV: dev
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
  except:
270
  - dev
271 272 273
  - master
  - /^nodeploy\/.*$/

274
  needs:
275
    - Test_N_Build:review
276 277

# Job to deploy to staging environment (WP-HX k8s cluster)
278 279
Staging:HX:
  extends: .deploy
280 281 282 283
  environment:
    name : wp-hx-staging
  only:
  - dev
284 285
  needs:
    - Test_N_Build
286 287
    - Nginx:Staging
    - Node:Staging
288

289 290 291
# Job to deploy to staging environment (WP-HL k8s cluster)
Staging:HL:
  extends: .deploy
292 293 294 295
  environment:
    name : wp-hh-staging
  only:
  - dev
296 297
  needs:
    - Test_N_Build
298 299
    - Nginx:Staging
    - Node:Staging
300

Kamal Dodiya's avatar
Kamal Dodiya committed
301
# Job to deploy to live environment (WP-HX k8s cluster)
302 303
Live:HX:
  extends: .deploy
Kamal Dodiya's avatar
Kamal Dodiya committed
304
  variables:
305 306
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-prod
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-prod
Kamal Dodiya's avatar
Kamal Dodiya committed
307 308 309 310
  environment:
    name : wp-hx-live
  only:
  - master
311 312
  needs:
    - Test_N_Build
313
    - Nginx:Live
314
    - Node:Live
Kamal Dodiya's avatar
Kamal Dodiya committed
315

316 317 318
# Job to deploy to live environment (WP-HL k8s cluster)
Live:HL:
  extends: .deploy
Kamal Dodiya's avatar
Kamal Dodiya committed
319
  variables:
320 321
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-prod
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-prod
Kamal Dodiya's avatar
Kamal Dodiya committed
322 323 324 325
  environment:
    name : wp-hh-live
  only:
  - master
326 327
  needs:
    - Test_N_Build
328
    - Nginx:Live
329
    - Node:Live
330

Jyothish's avatar
Jyothish committed
331
# Job to deploy to internal environment (WP-HX k8s cluster)
332 333
Internal:HX:
  extends: .deploy
Jyothish's avatar
Jyothish committed
334
  variables:
335 336
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-internal
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-internal
Jyothish's avatar
Jyothish committed
337 338 339
  environment:
    name : wp-hx-internal
  only:
340
  - master
341 342
  needs:
    - Test_N_Build:internal
343
    - Nginx:Internal
344
    - Node:Internal
Jyothish's avatar
Jyothish committed
345

346 347 348
# Job to deploy to internal environment (WP-HL k8s cluster)
Internal:HL:
  extends: .deploy
349
  variables:
350 351
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-internal
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-internal
352 353 354
  environment:
    name : wp-hh-internal
  only:
355
  - master
356 357
  needs:
    - Test_N_Build:internal
358
    - Nginx:Internal
359
    - Node:Internal
360

361 362
Review:HX:
  extends: .deploy-review
363 364 365
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
366
  environment:
367 368 369 370 371 372 373 374 375
    name: review/$CI_COMMIT_REF_SLUG
    url: http://$CI_COMMIT_REF_SLUG.review.ensembl.org
    on_stop: stop_review
    kubernetes:
      namespace: ensembl-dev
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
376
  needs:
377 378
    - Test_N_Build:review
    - Nginx:review
379
    - Node:review
380 381

stop_review:
382
  extends: .stop-review
383 384 385 386 387 388 389 390
  environment:
    name: review/$CI_COMMIT_REF_SLUG
    action: stop
    kubernetes:
      namespace: ensembl-dev
  except:
  - dev
  - master
Andrey Azov's avatar
Andrey Azov committed
391
  - /^nodeploy\/.*$/
392 393 394 395 396 397 398 399

SetupReview:HX:
  extends: .setup-review
  environment:
    name : wp-hx-dev-ing
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"'
      when: always