Merge branch 'staging'

.gitlab-ci.yml

@@ -11,6 +11,7 @@ stages:
 
 debug:
   stage: debug
+  image: alpine
   script:
     - export
 
@@ -22,21 +23,49 @@ setup:
   script:
     - docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
 
+dry.run.local:
+  stage: dry.run.local
+  image: praqma/helmsman:v3.5.1
+  script:
+    - helmsman --debug --group staging -f helmsman.yaml -f helmsman/staging.yaml --dry-run
+  only:
+    - staging
+
+
+dryrun:
+  stage: dryrun
+  image: praqma/helmsman:v3.5.1
+  script:
+    - helmsman --debug --group staging -f helmsman/token.yaml -f helmsman.yaml -f helmsman/staging.yaml --dry-run
+  only:
+    - staging
+
 staging:
   stage: staging
   image: praqma/helmsman:v3.5.1
   script:
-    - helmsman --apply --debug --group staging -f helmsman-token.yaml -f helmsman.yaml -f helmsman-staging.yaml
+    - helmsman --apply --debug --group staging -f helmsman/token.yaml -f helmsman.yaml -f helmsman/staging.yaml
+  only:
+    - staging
+
+staging.local:
+  stage: staging
+  image: praqma/helmsman:v3.5.1
+  script:
+    - helmsman --apply --debug --group staging -f helmsman.yaml -f helmsman/staging.yaml
   only:
     - staging
 
+
 production:
   stage: production
   image: praqma/helmsman:v3.5.1
   script:
-    - helmsman --apply --debug --group production -f helmsman-token.yaml -f helmsman.yaml helmsman-production.yaml
+    - helmsman --apply --debug --group production -f helmsman/token.yaml -f helmsman.yaml helmsman/production.yaml
   only:
     - master
+
+
 # testing:
 #   stage: testing
 #   image: python

Makefile

+BASEDIR = $(shell pwd)
+
+deploy.prod:
+	helmsman --apply --debug --group production -f helmsman/token.yaml -f helmsman.yaml -f helmsman/production.yaml
+deploy.staging:
+	helmsman --apply --debug --group staging -f helmsman/token.yaml -f helmsman.yaml -f helmsman/staging.yaml
+
+binder.deploy.prod:
+	helmsman --apply --debug --target binderhub-production -f helmsman.yaml -f helmsman/production.yaml --always-upgrade
+binder.deploy.staging:
+	helmsman --apply --debug --target binderhub-production -f helmsman.yaml --always-upgrade
+
+# Beta gpu enabled service
+
+
+beta.binder.deploy.prod:
+	helmsman --apply --debug --target binderhub-production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gpu.yaml --always-upgrade
+beta.binder.deploy.staging:
+	helmsman --apply --debug --target binderhub-production -f helmsman.yaml -f helmsman/gpu.yaml --always-upgrade
+
+# Beta gpu enabled service
+
+
+gpu.beta.binder.deploy.prod:
+	helmsman --apply --debug --target binderhub-production-gpu -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gpu.yaml --always-upgrade
+gpu.beta.binder.deploy.staging:
+	helmsman --apply --debug --target binderhub-production-gpu -f helmsman.yaml -f helmsman/gpu.yaml --always-upgrade
+
+# Dry runs for testing
+
+binder.deploy.prod.dry:
+	helmsman --debug --target binderhub-production -f helmsman.yaml -f helmsman/production.yaml --always-upgrade --dry-run
+
+beta.binder.deploy.prod.dry:
+	helmsman --debug --target binderhub-production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gpu.yaml --always-upgrade --dry-run
+
+gpu.beta.binder.deploy.prod.dry:
+	helmsman --debug --target binderhub-production-gpu -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gpu.yaml --always-upgrade --dry-run

binderhub/config.yaml

@@ -28,7 +28,12 @@ config:
     # networkPolicy:
     #   enabled: true
 #     # Good until this:
+cors: &cors
+  allowOrigin: '*'
+
 jupyterhub:
+  custom:
+    cors: *cors
   hub:
     baseUrl: /binderhub
     networkPolicy:
@@ -46,7 +51,7 @@ jupyterhub:
       limit: 2
       guarantee: 1
     memory:
-      limit: 4G
+      limit: 6G
       guarantee: 1G
   #   storage:
   #     type: none
@@ -89,11 +94,42 @@ dind:
       memory: 4Gi
 
 ingress:
-  # pathSuffix: ""
   enabled: true
   hosts:
     - ""
     - "localhost"
   annotations:
     kubernetes.io/ingress.class: nginx
-    nginx.ingress.kubernetes.io/proxy-body-size: 2m
\ No newline at end of file
+    kubernetes.io/tls-acme: "true"
+    cert-manager.k8s.io/acme-challenge-type: http01
+    cert-manager.io/cluster-issuer: letsencrypt-production
+    https:
+      enabled: true
+      type: nginx
+  tls:
+    - secretName: binder-bioimagearchive-org-cert
+      hosts:
+        - binder.bioimagearchive.org
+
+initContainers:
+  - name: git-clone-templates
+    image: alpine/git
+    args:
+      - clone
+      - --single-branch
+      - --branch=master
+      - --depth=1
+      - --
+      - https://github.com/bioimagearchive/k8s-jupyterhub
+      - /etc/binderhub/custom
+    securityContext:
+      runAsUser: 0
+    volumeMounts:
+      - name: custom-templates
+        mountPath: /etc/binderhub/custom
+extraVolumes:
+  - name: custom-templates
+    emptyDir: {}
+extraVolumeMounts:
+  - name: custom-templates
+    mountPath: /etc/binderhub/custom
\ No newline at end of file

binderhub/production/config.yaml

@@ -13,4 +13,16 @@ jupyterhub:
 ingress:
   enabled: true
   hosts:
-    - "binder.bioimagearchive.org"
\ No newline at end of file
+    - "binder.bioimagearchive.org"
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    kubernetes.io/tls-acme: "true"
+    cert-manager.k8s.io/acme-challenge-type: http01
+    cert-manager.io/cluster-issuer: letsencrypt-production
+    https:
+      enabled: true
+      type: nginx
+  tls:
+    - secretName: binder-bioimagearchive-org-cert
+      hosts:
+        - binder.bioimagearchive.org
\ No newline at end of file

binderhub/production/config_beta.yaml

 config:
   BinderHub:
-    hub_url: http://beta.binder.bioimagearchive.org/binderhub/
+    hub_url: beta.binder.bioimagearchive.org/binderhub/
     banner_message: |
-      <div style="text-align: center;">Beta service with more RAM and CPU</div>
+      <div style="text-align: center;">Beta service with more RAM and CPU  (no DIND atm)</div>
 jupyterhub:
   hub:
     baseUrl: /binderhub
@@ -15,6 +15,9 @@ ingress:
   enabled: true
   hosts:
     - "beta.binder.bioimagearchive.org"
+  tls:
+  annotations:
+    kubernetes.io/ingress.class: nginx
 
 dind:
   enabled: false
\ No newline at end of file

binderhub/production/config_beta_GPU.yaml

@@ -19,13 +19,16 @@ config:
   BinderHub:
     hub_url: http://gpu.beta.binder.bioimagearchive.org/binderhub/
     banner_message: |
-      <div style="text-align: center;">Beta service with more RAM and CPU and GPU Support (no DIND atm) </div>
+      <div style="text-align: center;">Beta service with more RAM and CPU and GPU Support </div>
 
 
 ingress:
   enabled: true
   hosts:
     - "gpu.beta.binder.bioimagearchive.org"
-
+  tls:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    
 dind:
   enabled: true
\ No newline at end of file

binderhub/staging/config.yaml

@@ -13,4 +13,16 @@ jupyterhub:
 ingress:
   enabled: true
   hosts:
-    - "staging.binder.bioimagearchive.org"
\ No newline at end of file
+    - "staging.binder.bioimagearchive.org"
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    kubernetes.io/tls-acme: "true"
+    cert-manager.k8s.io/acme-challenge-type: http01
+    cert-manager.io/cluster-issuer: letsencrypt-production
+    https:
+      enabled: true
+      type: nginx
+  tls:
+    - secretName: binder-bioimagearchive-org-cert
+      hosts:
+        - "staging.binder.bioimagearchive.org"
\ No newline at end of file

binderhub/staging/config_beta_GPU.yaml

@@ -4,7 +4,7 @@ jupyterhub:
   ingress:
     enabled: true
     hosts:
-      - gpu.staging.beta.binder.bioimagearchive.org
+      - staging.gpu.beta.binder.bioimagearchive.org
 
   profileList:
     - display_name: "GPU Server"
@@ -17,7 +17,7 @@ jupyterhub:
 
 config:
   BinderHub:
-    hub_url: http://gpu.staging.beta.binder.bioimagearchive.org/binderhub/
+    hub_url: http://staging.gpu.beta.binder.bioimagearchive.org/binderhub/
     banner_message: |
       <div style="text-align: center;">Beta service with more RAM and CPU and GPU Support</div>
 
@@ -25,4 +25,4 @@ config:
 ingress:
   enabled: true
   hosts:
-    - "gpu.staging.beta.binder.bioimagearchive.org"
+    - "staging.gpu.beta.binder.bioimagearchive.org"

cert-managment/cluster_issuer.yaml

+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-production
+spec:
+  acme:
+    # You must replace this email address with your own.
+    # Let's Encrypt will use this to contact you about expiring
+    # certificates, and issues related to your account.
+    email: ctr26@ebi.ac.uk
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource that will be used to store the account's private key.
+      name: letsencrypt-production
+    # Add a single challenge solver, HTTP01 using nginx
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
\ No newline at end of file

helmsman.yaml

@@ -31,6 +31,7 @@ metadata:
 helmRepos:
   jupyterhub: "https://jupyterhub.github.io/helm-chart/"
   daskgateway: "https://dask.org/dask-gateway-helm-repo/"
+  jetstack: "https://charts.jetstack.io"
 
 appsTemplates:
   binderhub: &binderhub
@@ -142,6 +143,7 @@ apps:
 
   binderhub-production-gpu:
     <<: *binderhub
+    enabled: false
     name: "binderhub-production-gpu"
     namespace: "binderhub-production-gpu"
     group: "production"
@@ -149,6 +151,7 @@ apps:
 
   binderhub-staging-gpu:
     <<: *binderhub
+    enabled: false
     name: "binderhub-staging-gpu"
     namespace: "binderhub-staging-gpu"
     group: "staging"
@@ -239,6 +242,23 @@ apps:
       # - "jupyterhub/persistentVolumes.yaml"
       - "jupyterhub/github.yaml"
       # - "jupyterhub/production/github.yaml"
+
+  cert-manager-production:
+    name: "cert-manager"
+    chart: "jetstack/cert-manager"
+    enabled: true
+    priority: 0
+    # timeout: 120
+    version: "v1.2.0"
+    group: "production"
+    namespace: "cert-manager"
+    set:
+      installCRDs: "true"
+      ingressShim.defaultIssuerKind: "ClusterIssuer"
+      ingressShim.defaultIssuerName: "letsencrypt-production"
+    hooks:
+      postUpgrade: "cert-managment/cluster_issuer.yaml"
+
 # -------------------- JUNK ----------------------------------
 # jupyterhub-test:
 #   valuesFiles:

helmsman-gpu.yaml → helmsman/gpu.yaml

 settings:
   kubeContext: gpu
 
-namespaces:
-  binderhub-staging-gpu:
-    protected: false
-  binderhub-production-gpu:
-    protected: false
+# namespaces:
+#   binderhub-staging-gpu:
+#     protected: false
+#   binderhub-production-gpu:
+#     protected: false
 
 apps:
   binderhub-production:
@@ -15,6 +15,7 @@ apps:
     - "binderhub/production/config_beta.yaml"
 
   binderhub-production-gpu:
+    enabled: true
     valuesFiles:
     - "binderhub/config.yaml"
     # - "binderhub/production/config.yaml"

helmsman-production.yaml → helmsman/production.yaml

@@ -15,7 +15,9 @@ namespaces:
     protected: false
   daskgateway-production:
     protected: false
-  # binderhub-staging-gpu:
-  #   protected: false
-  # binderhub-production-gpu:
-  #   protected: false
\ No newline at end of file
+  binderhub-production-gpu:
+    protected: false
+  binderhub-staging-gpu:
+    protected: false
+  cert-manager:
+    protected: false

helmsman-staging.yaml → helmsman/staging.yaml

@@ -19,3 +19,5 @@ namespaces:
     protected: false
   daskgateway-production:
     protected: true
+  cert-manager:
+    protected: true
\ No newline at end of file

helmsman-token.yaml → helmsman/token.yaml

@@ -4,4 +4,18 @@ settings:
   clusterURI: "https://kubernetes.default"
   bearerTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token"
 certificates:
-  caCrt: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
\ No newline at end of file
+  caCrt: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+# namespaces:
+#   jupyterhub-sandbox-staging:
+#     protected: true 
+#   jupyterhub-sandbox-production:
+#     protected: true
+#   binderhub-staging:
+#     protected: true 
+#   binderhub-production:
+#     protected: true
+#   daskgateway-staging:
+#     protected: true 
+#   daskgateway-production:
+#     protected: true
\ No newline at end of file

ingress/gpu_redirect.yaml

@@ -4,8 +4,19 @@ metadata:
   name: external-ip
   namespace: default
   annotations:
-    nginx.ingress.kubernetes.io/permanent-redirect: ""
+    # nginx.ingress.kubernetes.io/permanent-redirect: ""
+    kubernetes.io/ingress.class: nginx
+    certmanager.k8s.io/cluster-issuer: letsencrypt-productio
+    cert-manager.k8s.io/acme-challenge-type: http01
+    cert-manager.io/cluster-issuer: letsencrypt-production
 spec:
+  tls:
+  - hosts:
+    - beta.binder.bioimagearchive.org
+    secretName: beta-binder-bioimagearchive-org-tls
+  - hosts:
+    - gpu.beta.binder.bioimagearchive.org
+    secretName: gpu-beta-binder-bioimagearchive-org-tls
   rules:
   - host: "beta.binder.bioimagearchive.org"
     http:

tests/conftest.py

+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--binder-url",
+        help="Fully qualified URL to the binder installation"
+    )
+
+    parser.addoption(
+        "--hub-url",
+        help="Fully qualified URL to the hub installation"
+    )
+
+
+@pytest.fixture
+def binder_url(request):
+    return request.config.getoption("--binder-url").rstrip("/")
+
+
+@pytest.fixture
+def hub_url(request):
+    return request.config.getoption("--hub-url").rstrip("/")

tests/test_build.py

+from contextlib import contextmanager
+import json
+import subprocess
+import tempfile
+import time
+import os
+import sys
+
+import pytest
+import requests
+
+
+@contextmanager
+def push_dummy_gh_branch(repo, branch, keyfile):
+    """
+    Makes a dummy commit on a given github repo as a given branch
+
+    Requires that the branch not exist. keyfile should be an absolute path.
+
+    Should be used as a contextmanager, it will delete the branch & the
+    clone directory when done.
+    """
+
+    git_env = {'GIT_SSH_COMMAND': f"ssh -i {keyfile}"}
+
+    with tempfile.TemporaryDirectory() as gitdir:
+        subprocess.check_call(['git', 'clone', repo, gitdir], env=git_env)
+        branchfile = os.path.join(gitdir, 'branchname')
+        with open(branchfile, 'w') as f:
+            f.write(branch)
+        subprocess.check_call(['git', 'add', branchfile], cwd=gitdir)
+        subprocess.check_call(['git', 'commit', '-m', f'Dummy update for {branch}'], cwd=gitdir)
+        subprocess.check_call(
+            ['git', 'push', 'origin', f'HEAD:{branch}'],
+            env=git_env,
+            cwd=gitdir,
+        )
+
+        try:
+            yield
+        finally:
+            # Delete the branch so we don't clutter!
+            subprocess.check_call(
+                ['git', 'push', 'origin', f':{branch}'],
+                env=git_env,
+                cwd=gitdir,
+            )
+
+
+
+@pytest.mark.timeout(498)
+def test_build_binder(binder_url):
+    """
+    We can launch an image that we know hasn't been built
+    """
+    branch = str(time.time())
+    repo = 'binderhub-ci-repos/cached-minimal-dockerfile'
+
+    with push_dummy_gh_branch(
+        f"git@github.com:/{repo}.git",
+        branch,
+        os.path.abspath("secrets/binderhub-ci-repos-key"),
+    ):
+        build_url = binder_url + f"/build/gh/{repo}/{branch}"
+        print(f"building {build_url}")
+        r = requests.get(build_url, stream=True)
+        r.raise_for_status()
+        for line in r.iter_lines():
+            line = line.decode('utf8')
+            if line.startswith('data:'):
+                data = json.loads(line.split(':', 1)[1])
+                # include message output for debugging
+                if data.get('message'):
+                    sys.stdout.write(data['message'])
+                if data.get('phase') == 'ready':
+                    notebook_url = data['url']
+                    token = data['token']
+                    break
+        else:
+            # This means we never got a 'Ready'!
+            assert False
+
+        headers = {
+            'Authorization': f'token {token}'
+        }
+        r = requests.get(notebook_url + '/api', headers=headers)
+        assert r.status_code == 200
+        assert 'version' in r.json()
+
+        r = requests.post(notebook_url + '/api/shutdown', headers=headers)
+        assert r.status_code == 200

tests/test_http.py

+"""Basic HTTP tests to make sure things are running"""
+import pprint
+
+import pytest
+import requests
+
+
+def test_binder_up(binder_url):
+    """
+    Binder Hub URL is up & returning sensible text
+    """
+    resp = requests.get(binder_url)
+    assert resp.status_code == 200
+    assert 'GitHub' in resp.text
+
+
+def test_hub_health(hub_url):
+    """check JupyterHubHub health endpoint"""
+    resp = requests.get(hub_url + "/hub/health")
+    print(resp.text)
+    assert resp.status_code == 200
+
+
+def test_binder_health(binder_url):
+    """check BinderHub health endpoint"""
+    resp = requests.get(binder_url + "/health")
+    pprint.pprint(resp.json())
+    assert resp.status_code == 200
+
+
+# the proxy-patches pod can take up to 30 seconds
+# to register its route after a proxy restart
+@pytest.mark.flaky(reruns=3, reruns_delay=10)
+def test_hub_user_redirect(hub_url):
+    """Requesting a Hub URL for a non-running user"""
+    # this should *not* redirect for now,
+    resp = requests.get(hub_url + "/user/doesntexist")
+    assert resp.status_code == 404