Skip to content

GitLab

Commit ada70f64 authored by Craig Russell's avatar Craig Russell
Browse files

Merge branch 'login'

parents 8deef0f4 db39bae2
Hide whitespace changes
Inline Side-by-side
Showing with 956 additions and 84 deletions
Makefile
View file @ ada70f64
#!make
BASEDIR = $(shell pwd)
SOURCE:=$(shell source secrets.env)
# include .secrets
deploy.prod:
helmsman --apply --debug --group production -f helmsman/token.yaml -f helmsman.yaml -f helmsman/production.yaml
deploy.staging:
helmsman --apply --debug --group staging -f helmsman/token.yaml -f helmsman.yaml -f helmsman/staging.yaml
deploy.staging: helmsman --apply --debug --group staging -f helmsman/token.yaml -f helmsman.yaml -f helmsman/staging.yaml
binder.deploy.prod:
helmsman --apply --debug --target binderhub-production -f helmsman.yaml -f helmsman/production.yaml --always-upgrade
......@@ -18,6 +21,33 @@ beta.binder.deploy.prod:
beta.binder.deploy.staging:
helmsman --apply --debug --target binderhub-production -f helmsman.yaml -f helmsman/gpu.yaml --always-upgrade
# Alpha openstack service
persistent.alpha.binder.deploy.prod:
helmsman --apply --debug --target persistent-binderhub-production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/openstack.yaml --always-upgrade
persistent.alpha.binder.deploy.staging:
helmsman --apply --debug --target persistent-binderhub-production -f helmsman.yaml -f helmsman/openstack.yaml --always-upgrade
#
alpha.binder.deploy.prod:
helmsman --apply --debug --target binderhub-production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/openstack.yaml --always-upgrade
alpha.binder.deploy.staging:
helmsman --apply --debug --target binderhub-staging -f helmsman.yaml -f helmsman/staging.yaml -f helmsman/openstack.yaml --always-upgrade
alpha.deploy.prod:
helmsman --apply --debug --group production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/openstack.yaml --always-upgrade
alpha.cert.deploy.prod:
helmsman --apply --debug --target cert-manager-production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/openstack.yaml --always-upgrade
alpha.trow.deploy.prod:
helmsman --apply --debug --target trow -f helmsman.yaml -f helmsman/production.yaml -f helmsman/openstack.yaml --always-upgrade
alpha.registry.deploy.prod:
helmsman --apply --debug --target docker-registry -f helmsman.yaml -f helmsman/production.yaml -f helmsman/openstack.yaml --always-upgrade
alpha.ingress.deploy.prod:
helmsman --apply --debug --target ingress-nginx -f helmsman.yaml -f helmsman/production.yaml -f helmsman/openstack.yaml --always-upgrade
# Beta gpu enabled service
......@@ -25,6 +55,8 @@ gpu.beta.binder.deploy.prod:
helmsman --apply --debug --target binderhub-production-gpu -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gpu.yaml --always-upgrade
gpu.beta.binder.deploy.staging:
helmsman --apply --debug --target binderhub-production-gpu -f helmsman.yaml -f helmsman/gpu.yaml --always-upgrade
gpu.beta.triton.deploy.prod:
helmsman --apply --debug --target tritoninferenceserver -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gpu.yaml --always-upgrade
# Dry runs for testing
......@@ -36,3 +68,36 @@ beta.binder.deploy.prod.dry:
gpu.beta.binder.deploy.prod.dry:
helmsman --debug --target binderhub-production-gpu -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gpu.yaml --always-upgrade --dry-run
gpu.cert.deploy.prod:
helmsman --apply --debug --target cert-manager-production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gpu.yaml --always-upgrade
gke.binder.deploy.prod:
helmsman --apply --debug --target binderhub-production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gke.yaml --always-upgrade
gke.binder.deploy.nginx:
helmsman --apply --debug --target ingress-nginx -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gke.yaml --always-upgrade
gke.binder.deploy.staging:
helmsman --apply --debug --target binderhub-staging -f helmsman.yaml -f helmsman/staging.yaml -f helmsman/gke.yaml --always-upgrade
gke.deploy.prod:
helmsman --apply --debug --group production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gke.yaml --always-upgrade
gke.triton.deploy.prod:
helmsman --apply --debug --target tritoninferenceserver -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gke.yaml --always-upgrade
gke.persistent.alpha.binder.deploy.prod:
helmsman --apply --debug --target persistent-binderhub-production -f helmsman.yaml -f helmsman/production.yaml -f helmsman/gke.yaml --always-upgrade
gke.persistent.alpha.binder.deploy.staging:
helmsman --apply --debug --target persistent-binderhub-production -f helmsman.yaml -f helmsman/gke.yaml --always-upgrade
# CI_REGISTRY_USER = ${CI_REGISTRY_USER}
# CI_REGISTRY_PASSWORD = ${CI_REGISTRY_PASSWORD}
htpassword:
docker run --rm -ti xmartlabs/htpasswd ${CI_REGISTRY_USER} ${CI_REGISTRY_PASSWORD} > htpasswd_file
cat htpasswd_file
\ No newline at end of file
binderhub-persistent/config.yaml 0 → 100644
View file @ ada70f64
# binderhub:
# config:
# BinderHub:
# debug: true
# base_url: /
# hub_url: "persist.binder.bioimagearchive.org"
# image_prefix: ""
# use_registry: true
# build_image: jupyter/repo2docker:0.10.0
# jupyterhub:
# hub:
# # config:
# # # GitHubOAuthenticator:
# # # client_id:
# # # client_secret:
# # # oauth_callback_url: https://binder.bioimagearchive.org/hub/oauth_callback
# # JupyterHub:
# # authenticator_class: dummy
# # services:
# # binder:
# # oauth_client_id: "binder-oauth-client-dev"
# # url: "https://persist.binder.bioimagearchive.org"
# # apiToken: "b32f6db83b52afec0099e1154fdc852d053b36f724e03d657f38baf46334a662"
# # oauth_redirect_uri: "https://persist.binder.bioimagearchive.org/services/binder/oauth_callback"
# proxy:
# secretToken: "***REMOVED***"
# registry:
# username:
# password:
# ingress:
# enabled: true
# hosts:
# - "persist.binder.bioimagearchive.org"
# annotations:
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
# # cert-manager.k8s.io/acme-challenge-type: http01
# # cert-manager.io/cluster-issuer: letsencrypt-production
# https:
# enabled: true
# type: nginx
# # tls:
# # - secretName: persist-binder-bioimagearchive-org-cert
# # hosts:
# # - persist.binder.bioimagearchive.org
binderhub:
replicas:
1
dind:
# enabled: true
hostLibDir: /var/lib/dind/production/login
hostSocketDir: /var/run/dind/production/login
enabled: true
resources:
requests:
cpu: "2"
memory: 4Gi
limits:
cpu: "4"
memory: 6Gi
ingress:
enabled: false
service:
type: ClusterIP
config:
BinderHub:
auth_enabled: true
debug: true
# base_url: /
hub_url: "https://login.binder.bioimagearchive.org/"
# use only local docker images
image_prefix: ""
use_registry: true
# build_image: jupyter/repo2docker:0.10.0
jupyterhub:
cull:
# don't cull authenticated users
users: False
custom:
binderauth_enabled: true
hub:
extraConfig:
fuseConfig: |
from kubernetes import client
def modify_pod_hook(spawner, pod):
pod.spec.containers[0].security_context = client.V1SecurityContext(
privileged=True,
capabilities=client.V1Capabilities(
add=['SYS_ADMIN']
)
)
return pod
c.KubeSpawner.modify_pod_hook = modify_pod_hook
allowNamedServers: true
# change this value as you wish,
# or remove this line if you don't want to have any limit
namedServerLimitPerUser: 50
singleuser:
# to make notebook servers aware of hub
cmd: jupyterhub-singleuser
redirectToServer: false
baseUrl: /
config:
GitHubOAuthenticator:
client_id: ***REMOVED***
client_secret: ***REMOVED***
oauth_callback_url: https://login.binder.bioimagearchive.org/hub/oauth_callback
JupyterHub:
authenticator_class: github
services:
binder:
# this is the default value
oauth_no_confirm: true
oauth_client_id: "binder-oauth-client-dev"
url: "http://binder"
apiToken: "b32f6db83b52afec0099e1154fdc852d053b36f724e03d657f38baf46334a662"
oauth_redirect_uri: "https://login.binder.bioimagearchive.org/services/binder/oauth_callback"
ingress:
enabled: true
hosts:
- "login.binder.bioimagearchive.org"
tls:
- secretName: login-binder-bioimagearchive-org-cert
hosts:
- login.binder.bioimagearchive.org
annotations:
kubernetes.io/ingress.class: nginx
kubernetes.io/tls-acme: "true"
cert-manager.k8s.io/acme-challenge-type: http01
cert-manager.io/cluster-issuer: letsencrypt-production
https:
enabled: true
type: nginx
proxy:
secretToken: "***REMOVED***"
service:
type: ClusterIP
# registry:
# username:
# password:
# tls:
# - secretName: persist-binder-bioimagearchive-org-cert
# hosts:
# - persist.binder.bioimagearchive.org
# git: |
# from z2jh import get_config, get_secret, set_config_if_not_none
# # If we want to do some custom logic when we know whom have logged in...
# from oauthenticator.generic import GenericOAuthenticator
# class MyAuthenticator(GenericOAuthenticator):
# from tornado import gen
# @gen.coroutine
# def pre_spawn_start(self, user, spawner):
# auth_state = yield user.get_auth_state()
# if not auth_state:
# return
# # do stuff here... like...
# spawner.environment.update({'CAT_NAME': "missan"})
# "username=name\npassword=xxx"
\ No newline at end of file
binderhub/config.yaml
View file @ ada70f64
replicas: 2
# binderhub:
replicas: 1
pdb:
minAvailable: 1
......@@ -9,7 +10,6 @@ resources:
limits:
cpu: "2"
memory: 3Gi
config:
BinderHub:
template_path: /etc/binderhub/custom/binderhub/frontend-custom/template
......@@ -24,7 +24,9 @@ config:
# hub_url: http://binder.bioimagearchive.org/binderhub/
use_registry: true
image_prefix: bioimagearchive/binder-
build_image: jupyter/repo2docker:2021.01.0-35.gb6e451d
build_image: jupyter/repo2docker:2021.03.0-70.g43891a6
# build_image: jupyter/repo2docker:2021.01.0-35.gb6e451d
banner_message: |
<div style="text-align: center;">This is a public Beta and liable to downtime</div>
# build_image: "aicrowd/repo2docker"
......@@ -37,6 +39,9 @@ cors: &cors
allowOrigin: '*'
jupyterhub:
proxy:
service:
type: ClusterIP
cull:
users: True
custom:
......@@ -51,9 +56,21 @@ jupyterhub:
- bioimagearchive
scope:
- read:user
baseUrl: /binderhub
baseUrl: /jhub
networkPolicy:
enabled: true
extraConfig:
fuseConfig: |
from kubernetes import client
def modify_pod_hook(spawner, pod):
pod.spec.containers[0].security_context = client.V1SecurityContext(
privileged=True,
capabilities=client.V1Capabilities(
add=['SYS_ADMIN']
)
)
return pod
c.KubeSpawner.modify_pod_hook = modify_pod_hook
ingress:
enabled: true
hosts:
......@@ -63,7 +80,15 @@ jupyterhub:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/proxy-body-size: 2m
singleuser:
extraEnv:
GRANT_SUDO: "yes"
NOTEBOOK_ARGS: "--allow-root"
# JUPYTER_ENABLE_LAB: "true"
# uid: 0
# cmd: start-singleuser.sh
# cmd: jupyterhub-singleuser
# cmd: start-singleuser.sh
# cmd: null
cpu:
limit: 2
guarantee: 1
......@@ -80,13 +105,13 @@ jupyterhub:
# - name: nfs-bs-ftp
# mountPath: /home/jovyan/biostudies
# # readOnly: true
# profileList:
# - display_name: "GPU Server"
# description: "Spawns a notebook server with access to a GPU"
# kubespawner_override:
# extra_resource_limits:
# nvidia.com/gpu: "1"
# - display_name: "No GPU"
profileList:
- display_name: "GPU Server"
description: "Spawns a notebook server with access to a GPU"
kubespawner_override:
extra_resource_limits:
smarter-devices/fuse: "1"
# - display_name: "No GPU"
# description: "Spawns a notebook server with access to a GPU"
# proxy:
# # secretToken:
......@@ -101,14 +126,37 @@ jupyterhub:
# - ""
# - localhost
initContainers:
- name: git-clone-templates
image: alpine/git:latest
args:
- clone
- --single-branch
- --branch=master
- --depth=1
- --
- https://github.com/bioimagearchive/k8s-jupyterhub
- /etc/binderhub/custom
securityContext:
runAsUser: 0
volumeMounts:
- name: custom-templates
mountPath: /etc/binderhub/custom
extraVolumes:
- name: custom-templates
emptyDir: {}
extraVolumeMounts:
- name: custom-templates
mountPath: /etc/binderhub/custom
dind:
enabled: true
resources:
requests:
cpu: "1"
cpu: "2"
memory: 4Gi
limits:
cpu: "2"
cpu: "4"
memory: 6Gi
ingress:
......@@ -121,33 +169,14 @@ ingress:
kubernetes.io/tls-acme: "true"
cert-manager.k8s.io/acme-challenge-type: http01
cert-manager.io/cluster-issuer: letsencrypt-production
https:
enabled: true
type: nginx
certmanager.k8s.io/acme-http01-edit-in-place: "true"
# https:
# enabled: true
# type: nginx
tls:
- secretName: binder-bioimagearchive-org-cert
hosts:
- binder.bioimagearchive.org
initContainers:
- name: git-clone-templates
image: alpine/git
args:
- clone
- --single-branch
- --branch=master
- --depth=1
- --
- https://github.com/bioimagearchive/k8s-jupyterhub
- /etc/binderhub/custom
securityContext:
runAsUser: 0
volumeMounts:
- name: custom-templates
mountPath: /etc/binderhub/custom
extraVolumes:
- name: custom-templates
emptyDir: {}
extraVolumeMounts:
- name: custom-templates
mountPath: /etc/binderhub/custom
\ No newline at end of file
service:
type: ClusterIP
binderhub/deploy.sh
View file @ ada70f64
kubectl create namespace binderhub
helm install binderhub jupyterhub/binderhub --version=0.2.0-n217.h35366ea --namespace=binderhub -f secret.yaml -f config.yaml
helm upgrade binderhub jupyterhub/binderhub --version=0.2.0-n217.h35366ea -f secret.yaml -f config.yaml
\ No newline at end of file
helm upgrade binderhub jupyterhub/binderhub --version=0.2.0-n217.h35366ea -f secret.yaml -f config.yaml
helm upgrade binderhub jupyterhub/binderhub --version=0.2.0-n636.h8e3b5ac --namespace=binderhub -f binderhub/secret.yaml -f binderhub/config_basic.yaml --install --create-namespace
\ No newline at end of file
binderhub/production/config.yaml
View file @ ada70f64
config:
BinderHub:
auth_enabled: false
hub_url: https://binder.bioimagearchive.org/binderhub/
hub_url: https://binder.bioimagearchive.org/jhub/
jupyterhub:
hub:
baseUrl: /binderhub
extraConfig:
fuseConfig: |
from kubernetes import client
def modify_pod_hook(spawner, pod):
pod.spec.containers[0].security_context = client.V1SecurityContext(
privileged=True,
capabilities=client.V1Capabilities(
add=['SYS_ADMIN']
)
)
return pod
c.KubeSpawner.modify_pod_hook = modify_pod_hook
baseUrl: /jhub
config:
GitHubOAuthenticator:
oauth_callback_url: https://binder.bioimagearchive.org/binderhub/hub/oauth_callback
oauth_callback_url: https://binder.bioimagearchive.org/jhub/hub/oauth_callback
ingress:
enabled: true
hosts:
......@@ -25,10 +39,14 @@ ingress:
kubernetes.io/tls-acme: "true"
cert-manager.k8s.io/acme-challenge-type: http01
cert-manager.io/cluster-issuer: letsencrypt-production
https:
enabled: true
type: nginx
# https:
# enabled: true
# type: nginx
tls:
- secretName: binder-bioimagearchive-org-cert
hosts:
- binder.bioimagearchive.org
\ No newline at end of file
- "binder.bioimagearchive.org"
dind:
# enabled: true
hostLibDir: /var/lib/dind/production
hostSocketDir: /var/run/dind/production
\ No newline at end of file
binderhub/production/config_beta.yaml
View file @ ada70f64
# binderhub:
config:
BinderHub:
hub_url: https://beta.binder.bioimagearchive.org/binderhub/
debug: true
hub_url: https://beta.binder.bioimagearchive.org/jhub
banner_message: |
<div style="text-align: center;">Beta service with more RAM and CPU (no DIND atm)</div>
auth_enabled: false
jupyterhub:
# proxy:
# secretToken: "f61d5cb5bf61e6ca39894bfeb7c85bd75a79e3f8fbcf7a3054bd735a73c76737"
# https:
# enabled: true
custom:
binderauth_enabled: false
cull:
# don't cull authenticated users
users: False
# proxy:
# secretToken: 122d8b052d539dbab65d698ba1acab28b20fa6af83060c1479eceece9e32124b
hub:
baseUrl: /binderhub
# cookieSecret: "77a708e6aa1f1fadb67fb9e6479f390911b9e880f178d36a35d9301114f1767e"
# cookieSecret: "77a708e6aa1f1fadb67fb9e6479f390911b9e880f178d36a35d9301114f1767e"
config:
GitHubOAuthenticator:
client_id: "***REMOVED***"
client_secret: "***REMOVED***"
oauth_callback_url: https://beta.binder.bioimagearchive.org/jhub/hub/oauth_callback
allowed_organizations:
- bioimagearchive
scope:
- read:user
JupyterHub:
authenticator_class: dummy
# DummyAuthenticator:
# password: test
# oauth_callback_url: "https://beta.binder.bioimagearchive.org/jupyter/hub/oauth_callback"
# JupyterHub:
# authenticator_class: dummy
redirectToServer: false
baseUrl: /jhub
services:
binder:
# url: "http://binder"
# oauth_redirect_uri: "http://127.0.0.1:30123/services/binder/oauth_callback"
# oauth_client_id: "binder-oauth-client-test"
# apiToken: "b9c376305bb9ce2140f7f7953561e5c8687d40aa9e7a9c3580d52e1f91c4a27f"
oauth_no_confirm: true
# url: "http://binder"
# apiToken: "b9c376305bb9ce2140f7f7953561e5c8687d40aa9e7a9c3580d52e1f91c4a27f"
oauth_redirect_uri: https://beta.binder.bioimagearchive.org/oauth_callback
oauth_client_id: "binder-oauth-client-dev"
ingress:
enabled: true
hosts:
- beta.binder.bioimagearchive.org
singleuser:
# to make notebook servers aware of hub
cmd: jupyterhub-singleuser
# auth:
# type: github
# github:
# clientId: "***REMOVED***"
# clientSecret: "***REMOVED***"
# callbackUrl: "https://beta.binder.bioimagearchive.org/binderhub/hub/oauth_callback"
ingress:
enabled: true
......@@ -20,4 +74,103 @@ ingress:
kubernetes.io/ingress.class: nginx
dind:
enabled: true
\ No newline at end of file
enabled: false
# config:
# BinderHub:
# hub_url: https://beta.binder.bioimagearchive.org/binderhub
# auth_enabled: true
# jupyterhub:
# cull:
# # don't cull authenticated users
# users: False
# custom:
# binderauth_enabled: true
# hub:
# redirectToServer: false
# services:
# binder:
# oauth_no_confirm: true
# oauth_redirect_uri: "https://beta.binder.bioimagearchive.org/oauth_callback"
# oauth_client_id: "binder-oauth-client-test"
# ingress:
# enabled: true
# hosts:
# - beta.binder.bioimagearchive.org
# singleuser:
# # to make notebook servers aware of hub
# cmd: jupyterhub-singleuser
# binderhub:
# config:
# BinderHub:
# debug: true
# hub_url: http://<NodeIP>:30123
# # use only local docker images
# use_registry: false
# service:
# type: NodePort
# nodePort: 30124
# imageCleaner:
# enabled: false
# jupyterhub:
# debug:
# enabled: true
# hub:
# cookieSecret: "77a708e6aa1f1fadb67fb9e6479f390911b9e880f178d36a35d9301114f1767e"
# services:
# binder:
# url: http://<NodeIP>:30124
# oauth_redirect_uri: "http://<NodeIP>:30123/services/binder/oauth_callback"
# oauth_client_id: "binder-oauth-client-test"
# apiToken: "b9c376305bb9ce2140f7f7953561e5c8687d40aa9e7a9c3580d52e1f91c4a27f"