.gitlab-ci.yml 11.7 KB
Newer Older
1 2
# CI/CD Pipeline
# dev branch -> staging
3
# master branch -> internal (Custom Downloads)
4 5
# master branch -> live

6 7 8
# include template to setup review app
include: '/gitlab-ci-templates/.setup-review-template.yaml'

9
# Define stages for the CI/CD pipeline
10
stages:
11
  - setup
12 13 14 15
  - test
  - test_build_static
  - build_docker_images
  - deploy
Andrey Azov's avatar
Andrey Azov committed
16

17
variables:
18 19
  CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}
  CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}
20
  KUBE_CONTEXT: ens-dev-ctx
21
  DEPLOYENV: dev
22
  ENVIRONMENT: production
23
  DOCKER_TLS_CERTDIR: ""
Andrey Azov's avatar
Andrey Azov committed
24

25 26 27
# Template to build static assets
.build-static:
  stage: test_build_static
28
  image: node:14.17.0
29 30 31

  before_script:
  - cd src/ensembl
32
  - npm install --global npm@7.12.1
33 34 35 36 37 38
  - npm ci

  script:
  - npm run test
  - npm run build

39 40 41
  after_script:
    - echo "BUILD_JOB_ID=${CI_JOB_ID}" >> build_static.env

42
  artifacts:
43
    name: build_artifacts
44
    paths:
45 46 47 48 49
      - src/ensembl/dist/
      - src/ensembl/package.json
      - src/ensembl/package-lock.json
    reports:
      dotenv: build_static.env
50 51 52

# Template to build nginx docker image for WP k8s cluster
# WP k8s cluster uses ingress
53
.build-nginx:
54 55 56 57 58 59 60 61 62 63 64 65 66 67
  image: docker

  services:
    - docker:dind

  stage: build_docker_images

  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  script:
    - apk update && apk add git
    - git clone --depth 1 https://github.com/Ensembl/ensembl-2020-static-assests.git
    - git clone https://gitlab.ebi.ac.uk/kamal/ensembl-client-nginx.git
68
    - git -C ensembl-client-nginx checkout migration/ssr
69 70 71 72 73
    - docker build -t ${CONTAINER_IMAGE} -f ensembl-client-nginx/Dockerfile --no-cache .
    - docker push ${CONTAINER_IMAGE}
    - docker rmi ${CONTAINER_IMAGE}
    - docker logout $CI_REGISTRY

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
.publish_assets:
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.5
  stage: deploy
  environment:
    name : jobs/k8s-job

  before_script:
    - git clone https://gitlab.ebi.ac.uk/ensembl-web/ensembl-k8s-manifests.git
    - git -C ensembl-k8s-manifests/ checkout wp-k8s
    - cd ensembl-k8s-manifests

  script:
    - sed -i "s#<PROJECT_ID>#${CI_PROJECT_ID}#g" static_assets_copy_job.yaml
    - sed -i "s#<JOB_ID>#${BUILD_JOB_ID}#g" static_assets_copy_job.yaml
    - kustomize edit set namesuffix -- -${CI_PROJECT_ID}-${BUILD_JOB_ID}
    - kubectl apply -k .

91 92 93
# Template to build node docker image for WP k8s cluster
# WP k8s cluster uses ingress
.build-node:
94 95 96 97 98 99 100 101 102 103 104
  image: docker

  services:
    - docker:dind

  stage: build_docker_images

  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  script:
105 106 107 108
    - docker build -t ${CONTAINER_NODE_IMAGE} -f src/ensembl/Dockerfile.node --no-cache .
    - docker push ${CONTAINER_NODE_IMAGE}
    - docker rmi ${CONTAINER_NODE_IMAGE}
    - docker logout $CI_REGISTRY
109

110
# Template to deploy to WP k8s cluster
111
.deploy:
112
  stage: deploy
113
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
114
  script:
115 116 117 118 119 120
  - git clone https://gitlab.ebi.ac.uk/ensembl-web/ensembl-k8s-manifests.git
  - git -C ensembl-k8s-manifests/ checkout wp-k8s
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_NODE_IMAGE}#g" ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
  - kubectl apply -f ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - kubectl apply -f ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
121

122
# Template to deploy review-app to WP k8s cluster
123
.deploy-review:
124
  stage: deploy
125
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
126
  script:
127 128 129 130 131 132 133 134
  - git clone https://gitlab.ebi.ac.uk/ensembl-web/ensembl-k8s-manifests.git
  - git -C ensembl-k8s-manifests/ checkout wp-k8s-review
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_IMAGE}#g" ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - sed -i "s#<DEPLOYMENT_ENV>#${CI_COMMIT_REF_SLUG}#g" ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
  - sed -i "s#<DOCKER_IMAGE>#${CONTAINER_NODE_IMAGE}#g" ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
  - kubectl apply -f ensembl-k8s-manifests/ensembl_client_nginx_deployment.yaml
  - kubectl apply -f ensembl-k8s-manifests/ensembl_client_node_deployment.yaml
135 136

# Template for stopping review app - Do cleanup here 
137
.stop-review:
138
  stage: deploy
139
  image: dockerhub.ebi.ac.uk/ensembl-web/deploy-tools:0.1
140 141 142 143 144 145 146 147
  variables:
    GIT_STRATEGY: none
  when: manual
  script:
  - kubectl delete ingress ensembl-client-${CI_COMMIT_REF_SLUG}-ingress
  - kubectl delete svc ensembl-client-${CI_COMMIT_REF_SLUG}-svc
  - kubectl delete deploy ensembl-client-${CI_COMMIT_REF_SLUG}-deployment

148
Test:
Andrey Azov's avatar
Andrey Azov committed
149
  image: node:14.16.0
150
  stage: test
Andrey Azov's avatar
Andrey Azov committed
151

152 153 154 155 156
  before_script:
  - cd src/ensembl
  - npm ci

  script:
157
  - npm run check-types
158 159
  - npm run test

160 161
  only:
  - /^nodeploy\/.*$/
162 163 164 165
  except:
  - dev
  - master

166 167 168
# Job to build static asset for staging and live environment
# dev branch -> Staging
# master branch -> Live
169
Test_N_Build:
170
  extends: .build-static
171

172 173 174 175
  variables:
    GOOGLE_ANALYTICS_KEY: ${GA_KEY}
    ENVIRONMENT: production
    API_HOST: ""
176 177 178 179 180

  only:
  - dev
  - master

181
# Job to build static asset for internal environment
182
# master branch -> Internal
183
Test_N_Build:internal:
184
  extends: .build-static
185

186 187 188 189
  variables:
    GOOGLE_ANALYTICS_KEY: ${GA_KEY}
    ENVIRONMENT: internal
    API_HOST: ""
190 191

  only:
192
  - master
193

194
Test_N_Build:review:
195 196 197 198
  extends: .build-static
  variables:
    ENVIRONMENT: development
    API_HOST: ""
199

200 201 202 203
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
204

205 206 207 208
# Job to build nginx docker image for staging environment
# dev branch -> Staging
Nginx:Staging:
  extends: .build-nginx
209 210
  only:
  - dev
211 212
  needs:
    - Test_N_Build
213

214
# Job to build node docker image for staging environment
215
# dev branch -> Staging
216 217
Node:Staging:
  extends: .build-node
218 219
  only:
  - dev
220 221
  needs:
    - Test_N_Build
222 223 224

# Job to build nginx docker image for live environment
# master branch -> Live
225 226
Nginx:Live:
  extends: .build-nginx
227
  variables:
228
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-prod
229 230 231 232 233 234 235 236 237 238 239
  only:
  - master
  needs:
    - Test_N_Build

# Job to build node docker image for live environment
# master branch -> Live
Node:Live:
  extends: .build-node
  variables:
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-prod
240
  only:
241
  - master
242 243
  needs:
    - Test_N_Build
244

245
# Job to build nginx docker image for internal environment at WP
246
# master branch -> internal
247 248
Nginx:Internal:
  extends: .build-nginx
249 250
  variables:
    DEPLOYENV: internal
251
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-internal
252
  only:
253
  - master
254 255
  needs:
    - Test_N_Build:internal
256

257
# Job to build node docker image for internal environment at WP
258
# master branch -> internal
259 260
Node:Internal:
  extends: .build-node
Jyothish's avatar
Jyothish committed
261 262
  variables:
    DEPLOYENV: internal
263
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-internal
Jyothish's avatar
Jyothish committed
264
  only:
265
  - master
266 267
  needs:
    - Test_N_Build:internal
Jyothish's avatar
Jyothish committed
268

269 270
# Job to build nginx docker image for review environment at WP
# all other branch -> dev
271
Nginx:review:
272
  extends: .build-nginx
273 274
  variables:
    DEPLOYENV: dev
275
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
276 277 278 279
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
280

281
  needs:
282
    - Test_N_Build:review
283

284 285 286 287
# Job to build node docker image for review environment at WP
# all other branch -> dev
Node:review:
  extends: .build-node
288
  variables:
289 290 291
    DEPLOYENV: dev
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
  except:
292
  - dev
293 294 295
  - master
  - /^nodeploy\/.*$/

296
  needs:
297
    - Test_N_Build:review
298 299

# Job to deploy to staging environment (WP-HX k8s cluster)
300 301
Staging:HX:
  extends: .deploy
302 303 304 305
  environment:
    name : wp-hx-staging
  only:
  - dev
306 307
  needs:
    - Test_N_Build
308 309
    - Nginx:Staging
    - Node:Staging
310

311 312 313
Pub:Staging-HX:
  extends: .publish_assets
  environment:
314 315 316
    name : hx-jobs/staging
    kubernetes:
      namespace: ensembl-stage
317 318 319 320 321 322
  only:
    - dev
  needs:
    - Test_N_Build
    - Node:Staging

323 324 325
# Job to deploy to staging environment (WP-HL k8s cluster)
Staging:HL:
  extends: .deploy
326 327 328 329
  environment:
    name : wp-hh-staging
  only:
  - dev
330 331
  needs:
    - Test_N_Build
332 333
    - Nginx:Staging
    - Node:Staging
334

335 336 337
Pub:Staging-HL:
  extends: .publish_assets
  environment:
338 339 340
    name : hl-jobs/staging
    kubernetes:
      namespace: ensembl-stage
341 342 343 344 345 346
  only:
    - dev
  needs:
    - Test_N_Build
    - Node:Staging

Kamal Dodiya's avatar
Kamal Dodiya committed
347
# Job to deploy to live environment (WP-HX k8s cluster)
348 349
Live:HX:
  extends: .deploy
Kamal Dodiya's avatar
Kamal Dodiya committed
350
  variables:
351 352
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-prod
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-prod
Kamal Dodiya's avatar
Kamal Dodiya committed
353 354 355 356
  environment:
    name : wp-hx-live
  only:
  - master
357 358
  needs:
    - Test_N_Build
359
    - Nginx:Live
360
    - Node:Live
Kamal Dodiya's avatar
Kamal Dodiya committed
361

362 363 364
Pub:Live-HX:
  extends: .publish_assets
  environment:
365 366 367
    name : hx-jobs/prod
    kubernetes:
      namespace: ensembl-prod
368 369 370 371 372 373
  only:
  - master
  needs:
    - Test_N_Build
    - Node:Live

374 375 376
# Job to deploy to live environment (WP-HL k8s cluster)
Live:HL:
  extends: .deploy
Kamal Dodiya's avatar
Kamal Dodiya committed
377
  variables:
378 379
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-prod
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-prod
Kamal Dodiya's avatar
Kamal Dodiya committed
380 381 382 383
  environment:
    name : wp-hh-live
  only:
  - master
384 385
  needs:
    - Test_N_Build
386
    - Nginx:Live
387
    - Node:Live
388

389 390 391
Pub:Live-HL:
  extends: .publish_assets
  environment:
392 393 394
    name : hl-jobs/prod
    kubernetes:
      namespace: ensembl-prod
395 396 397 398 399 400
  only:
  - master
  needs:
    - Test_N_Build
    - Node:Live

Jyothish's avatar
Jyothish committed
401
# Job to deploy to internal environment (WP-HX k8s cluster)
402 403
Internal:HX:
  extends: .deploy
Jyothish's avatar
Jyothish committed
404
  variables:
405 406
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-internal
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-internal
Jyothish's avatar
Jyothish committed
407 408 409
  environment:
    name : wp-hx-internal
  only:
410
  - master
411 412
  needs:
    - Test_N_Build:internal
413
    - Nginx:Internal
414
    - Node:Internal
Jyothish's avatar
Jyothish committed
415

416 417 418
Pub:Internal-HX:
  extends: .publish_assets
  environment:
419 420 421
    name : hx-jobs/internal
    kubernetes:
      namespace: ensembl-internal
422 423 424 425 426 427
  only:
  - master
  needs:
    - Test_N_Build:internal
    - Node:Internal

428 429 430
# Job to deploy to internal environment (WP-HL k8s cluster)
Internal:HL:
  extends: .deploy
431
  variables:
432 433
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-internal
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-internal
434 435 436
  environment:
    name : wp-hh-internal
  only:
437
  - master
438 439
  needs:
    - Test_N_Build:internal
440
    - Nginx:Internal
441
    - Node:Internal
442

443 444 445
Pub:Internal-HL:
  extends: .publish_assets
  environment:
446 447 448
    name : hl-jobs/internal
    kubernetes:
      namespace: ensembl-internal
449 450 451 452 453 454
  only:
  - master
  needs:
    - Test_N_Build:internal
    - Node:Internal

455 456
Review:HX:
  extends: .deploy-review
457 458 459
  variables:
    CONTAINER_IMAGE: ${CI_REGISTRY_IMAGE}:nginx-${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
    CONTAINER_NODE_IMAGE: ${CI_REGISTRY_IMAGE}:node-${CI_COMMIT_SHORT_SHA}-${CI_COMMIT_REF_SLUG}
460
  environment:
461 462 463 464 465 466 467 468 469
    name: review/$CI_COMMIT_REF_SLUG
    url: http://$CI_COMMIT_REF_SLUG.review.ensembl.org
    on_stop: stop_review
    kubernetes:
      namespace: ensembl-dev
  except:
  - dev
  - master
  - /^nodeploy\/.*$/
470
  needs:
471 472
    - Test_N_Build:review
    - Nginx:review
473
    - Node:review
474 475

stop_review:
476
  extends: .stop-review
477 478 479 480 481 482 483 484
  environment:
    name: review/$CI_COMMIT_REF_SLUG
    action: stop
    kubernetes:
      namespace: ensembl-dev
  except:
  - dev
  - master
Andrey Azov's avatar
Andrey Azov committed
485
  - /^nodeploy\/.*$/
486 487 488 489 490 491 492 493

SetupReview:HX:
  extends: .setup-review
  environment:
    name : wp-hx-dev-ing
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"'
      when: always