diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 868d0e653..aba3bea8c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -95,7 +95,7 @@ jobs: - name: Install Helm Chart run: | helm dep update helm-chart/renku-notebooks - helm install renku-notebooks helm-chart/renku-notebooks -f test-values.yaml --wait --timeout 5m0s + helm install renku-notebooks helm-chart/renku-notebooks -f test-values.yaml --wait --timeout 15m0s - name: Helm Test run: | helm test renku-notebooks --timeout 60m0s --logs diff --git a/git-clone/clone.sh b/git-clone/clone.sh index aea56a70a..1f770447d 100644 --- a/git-clone/clone.sh +++ b/git-clone/clone.sh @@ -24,12 +24,12 @@ pat='p\/([^\/]*?)\.git' REPOSITORY_NAME="${BASH_REMATCH[1]}" # Wait in case gitlab is temporarily unavailable -curl $GIT_HOST +curl $GIT_URL while [ $? != 0 ] do - echo "Waiting for git server to become visible at $GIT_HOST" + echo "Waiting for git server to become visible at $GIT_URL" sleep 5 - curl $GIT_HOST + curl $GIT_URL done echo "Git server found" diff --git a/git-https-proxy/Dockerfile b/git-https-proxy/Dockerfile index b6a3556e0..3399826f4 100644 --- a/git-https-proxy/Dockerfile +++ b/git-https-proxy/Dockerfile @@ -5,4 +5,4 @@ LABEL maintainer="Swiss Data Science Center " COPY package.json package-lock.json mitmproxy.js ./ RUN npm ci && npm cache clean --force -CMD ["node", "/mitmproxy.js"] +CMD ["node", "--use-openssl-ca", "/mitmproxy.js"] diff --git a/helm-chart/renku-notebooks/requirements.lock b/helm-chart/renku-notebooks/requirements.lock index 9ea545e2e..6b43934bb 100644 --- a/helm-chart/renku-notebooks/requirements.lock +++ b/helm-chart/renku-notebooks/requirements.lock @@ -2,8 +2,11 @@ dependencies: - name: amalthea repository: https://swissdatasciencecenter.github.io/helm-charts/ version: 0.2.2 +- name: certificates + repository: https://swissdatasciencecenter.github.io/helm-charts/ + version: 0.0.1 - name: dlf-chart repository: https://swissdatasciencecenter.github.io/datashim/ version: 0.1.1-renku-1 -digest: sha256:2623679a03eb93eedafd5149fcea45e1fcaf7b14b3ad18d0e12a24f47280fa90 -generated: "2022-01-19T17:41:34.437304+01:00" +digest: sha256:6e0f563928d013dab0568ef4e23a6b761a8fd39c6eda22797d17e759320bac1b +generated: "2022-01-25T22:59:52.005894+01:00" diff --git a/helm-chart/renku-notebooks/requirements.yaml b/helm-chart/renku-notebooks/requirements.yaml index b45f4cfc9..1d7cf67c3 100644 --- a/helm-chart/renku-notebooks/requirements.yaml +++ b/helm-chart/renku-notebooks/requirements.yaml @@ -1,7 +1,10 @@ dependencies: - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.2.2" + version: 0.2.2 +- name: certificates + version: "0.0.1" + repository: "https://swissdatasciencecenter.github.io/helm-charts/" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.1.1-renku-1" diff --git a/helm-chart/renku-notebooks/templates/configmap.yaml b/helm-chart/renku-notebooks/templates/configmap.yaml index 3f544ad0c..8287f687f 100644 --- a/helm-chart/renku-notebooks/templates/configmap.yaml +++ b/helm-chart/renku-notebooks/templates/configmap.yaml @@ -16,20 +16,6 @@ data: --- apiVersion: v1 kind: ConfigMap -metadata: - name: hub-config-spawner - labels: - app: {{ template "notebooks.name" . }} - chart: {{ template "notebooks.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: -{{- if .Values.sentry.dsn }} - SENTRY_DSN: {{ .Values.sentry.dsn | quote }} -{{- end }} ---- -apiVersion: v1 -kind: ConfigMap metadata: name: notebook-helper-scripts labels: diff --git a/helm-chart/renku-notebooks/templates/statefulset.yaml b/helm-chart/renku-notebooks/templates/statefulset.yaml index cc0e8ff71..7e727637c 100644 --- a/helm-chart/renku-notebooks/templates/statefulset.yaml +++ b/helm-chart/renku-notebooks/templates/statefulset.yaml @@ -104,6 +104,11 @@ spec: - name: SENTRY_ENV value: {{ .Values.sentry.env | quote }} {{ end }} + - name: CERTIFICATES_IMAGE + value: "{{ .Values.global.certificates.image.repository }}:{{ .Values.global.certificates.image.tag }}" + - name: CUSTOM_CA_CERTS_SECRETS + value: | + {{- .Values.global.certificates.customCAs | toYaml | nindent 16 }} {{- with .Values.sessionNodeSelector }} - name: SESSION_NODE_SELECTOR value: | @@ -124,6 +129,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.namespace + {{- include "certificates.env.python" . | nindent 12 }} - name: ENFORCE_CPU_LIMITS value: {{ .Values.enforceCPULimits | quote }} - name: S3_MOUNTS_ENABLED @@ -135,6 +141,7 @@ spec: volumeMounts: - name: server-options mountPath: /etc/renku-notebooks/server_options + {{- include "certificates.volumeMounts.system" . | nindent 12 }} livenessProbe: httpGet: path: /health @@ -148,6 +155,7 @@ spec: resources: {{ toYaml .Values.resources | indent 12 }} initContainers: + {{- include "certificates.initContainer" . | nindent 8 }} - name: k8s-resource-schema-migrations image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} @@ -173,6 +181,7 @@ spec: - name: server-options configMap: name: {{ template "notebooks.fullname" . }}-options + {{- include "certificates.volumes" . | nindent 8 }} serviceAccountName: {{ if .Values.rbac.create }}"{{ template "notebooks.fullname" . }}"{{ else }}"{{ .Values.rbac.serviceAccountName }}"{{ end }} {{- with .Values.nodeSelector }} diff --git a/helm-chart/renku-notebooks/values.yaml b/helm-chart/renku-notebooks/values.yaml index f118daa22..da895e6a6 100644 --- a/helm-chart/renku-notebooks/values.yaml +++ b/helm-chart/renku-notebooks/values.yaml @@ -10,6 +10,17 @@ global: domain: anonymousSessions: enabled: false + ## Specify a secret that containes the certificate + ## if you would like to use a custom CA. The key for the secret + ## should have the .crt extension otherwise it is ignored. The + ## keys across all secrets are mounted as files in one location so + ## the keys across all secrets have to be unique. + certificates: + image: + repository: renku/certificates + tag: "0.0.1" + customCAs: [] + # - secret: amalthea: scope: diff --git a/renku_notebooks/api/amalthea_patches/__init__.py b/renku_notebooks/api/amalthea_patches/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/renku_notebooks/api/amalthea_patches/autosave.py b/renku_notebooks/api/amalthea_patches/autosave.py new file mode 100644 index 000000000..523a1c51d --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/autosave.py @@ -0,0 +1,61 @@ +def main(): + patches = [] + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/lifecycle", + "value": { + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "/usr/local/bin/pre-stop.sh", + "||", + "true", + ] + } + } + }, + } + ], + } + ) + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/volumes/-", + "value": { + "name": "notebook-helper-scripts-volume", + "configMap": { + "name": "notebook-helper-scripts", + "defaultMode": 493, + }, + }, + } + ], + } + ) + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/volumeMounts/-", + "value": { + "mountPath": "/usr/local/bin/pre-stop.sh", + "name": "notebook-helper-scripts-volume", + "subPath": "pre-stop.sh", + }, + } + ], + } + ) + return patches diff --git a/renku_notebooks/api/amalthea_patches/general.py b/renku_notebooks/api/amalthea_patches/general.py new file mode 100644 index 000000000..453b13957 --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/general.py @@ -0,0 +1,111 @@ +from flask import current_app + +from ..classes.user import RegisteredUser + + +def session_tolerations(): + patches = [] + tolerations = [ + { + "key": f"{current_app.config['RENKU_ANNOTATION_PREFIX']}dedicated", + "operator": "Equal", + "value": "user", + "effect": "NoSchedule", + }, + *current_app.config["SESSION_TOLERATIONS"], + ] + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/tolerations", + "value": tolerations, + } + ], + } + ) + return patches + + +def session_affinity(): + return [ + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/affinity", + "value": current_app.config["SESSION_AFFINITY"], + } + ], + } + ] + + +def session_node_selector(): + return [ + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/nodeSelector", + "value": current_app.config["SESSION_NODE_SELECTOR"], + } + ], + } + ] + + +def test(server): + """RFC 6901 patches support test statements that will cause the whole patch + to fail if the test statements are not correct. This is used to ensure that the + order of containers in the amalthea manifests is what the notebook service expects.""" + patches = [] + container_names = ( + current_app.config["AMALTHEA_CONTAINER_ORDER_REGISTERED_SESSION"] + if type(server._user) is RegisteredUser + else current_app.config["AMALTHEA_CONTAINER_ORDER_ANONYMOUS_SESSION"] + ) + for container_ind, container_name in enumerate(container_names): + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "test", + "path": ( + "/statefulset/spec/template/spec" + f"/containers/{container_ind}/name" + ), + "value": container_name, + } + ], + } + ) + return patches + + +def oidc_unverified_email(server): + patches = [] + if type(server._user) is RegisteredUser: + # modify oauth2 proxy to accept users whose email has not been verified + # usually enabled for dev purposes + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/1/env/-", + "value": { + "name": "OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL", + "value": current_app.config["OIDC_ALLOW_UNVERIFIED_EMAIL"], + }, + }, + ], + } + ) + return patches diff --git a/renku_notebooks/api/amalthea_patches/git_proxy.py b/renku_notebooks/api/amalthea_patches/git_proxy.py new file mode 100644 index 000000000..4dc9939c2 --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/git_proxy.py @@ -0,0 +1,58 @@ +from flask import current_app + +from ..classes.user import RegisteredUser +from .utils import get_certificates_volume_mounts + + +def main(server): + etc_cert_volume_mount = get_certificates_volume_mounts( + custom_certs=False, + etc_certs=True, + read_only_etc_certs=True, + ) + patches = [] + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/-", + "value": { + "image": current_app.config["GIT_HTTPS_PROXY_IMAGE"], + "name": "git-proxy", + "env": [ + { + "name": "REPOSITORY_URL", + "value": server.gl_project.http_url_to_repo, + }, + {"name": "MITM_PROXY_PORT", "value": "8080"}, + {"name": "HEALTH_PORT", "value": "8081"}, + { + "name": "GITLAB_OAUTH_TOKEN", + "value": server._user.git_token, + }, + { + "name": "ANONYMOUS_SESSION", + "value": ( + "false" + if type(server._user) is RegisteredUser + else "true" + ), + }, + ], + "livenessProbe": { + "httpGet": {"path": "/health", "port": 8081}, + "initialDelaySeconds": 3, + }, + "readinessProbe": { + "httpGet": {"path": "/health", "port": 8081}, + "initialDelaySeconds": 3, + }, + "volumeMounts": etc_cert_volume_mount, + }, + } + ], + } + ) + return patches diff --git a/renku_notebooks/api/amalthea_patches/git_sidecar.py b/renku_notebooks/api/amalthea_patches/git_sidecar.py new file mode 100644 index 000000000..0e387a30f --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/git_sidecar.py @@ -0,0 +1,80 @@ +def main(): + patches = [] + # patches.append( + # { + # "type": "application/json-patch+json", + # "patch": [ + # { + # "op": "add", + # "path": "/statefulset/spec/template/spec/containers/-", + # "value": { + # "image": current_app.config["GIT_RPC_SERVER_IMAGE"], + # "name": "git-sidecar", + # # Do not expose this until access control is in place + # # "ports": [ + # # { + # # "containerPort": 4000, + # # "name": "git-port", + # # "protocol": "TCP", + # # } + # # ], + # "env": [ + # { + # "name": "MOUNT_PATH", + # "value": f"/work/{self.gl_project.path}", + # } + # ], + # "resources": {}, + # "securityContext": { + # "allowPrivilegeEscalation": False, + # "fsGroup": 100, + # "runAsGroup": 100, + # "runAsUser": 1000, + # }, + # "volumeMounts": [ + # { + # "mountPath": f"/work/{self.gl_project.path}/", + # "name": "workspace", + # "subPath": f"{self.gl_project.path}/", + # } + # ], + # # Enable readiness and liveness only when control is in place + # # "livenessProbe": { + # # "httpGet": {"port": 4000, "path": "/"}, + # # "periodSeconds": 30, + # # # delay should equal periodSeconds x failureThreshold + # # # from readiness probe values + # # "initialDelaySeconds": 600, + # # }, + # # the readiness probe will retry 36 times over 360 seconds to see + # # if the pod is ready to accept traffic - this gives the user session + # # a maximum of 360 seconds to setup the git sidecar and clone the repo + # # "readinessProbe": { + # # "httpGet": {"port": 4000, "path": "/"}, + # # "periodSeconds": 10, + # # "failureThreshold": 60, + # # }, + # }, + # } + # ], + # } + # ) + # We can add this to expose the git side-car once it's protected. + # patches.append( + # { + # "type": "application/json-patch+json", + # "patch": [ + # { + # "op": "add", + # "path": "/service/spec/ports/-", + # "value": { + # "name": "git-service", + # "port": 4000, + # "protocol": "TCP", + # "targetPort": 4000, + # }, + # } + # ], + # } + # ) + return patches diff --git a/renku_notebooks/api/amalthea_patches/init_containers.py b/renku_notebooks/api/amalthea_patches/init_containers.py new file mode 100644 index 000000000..e4b50ecbe --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/init_containers.py @@ -0,0 +1,148 @@ +from flask import current_app +from kubernetes import client + +from ..classes.user import RegisteredUser +from .utils import get_certificates_volume_mounts + + +def git_clone(server): + etc_cert_volume_mount = get_certificates_volume_mounts( + custom_certs=False, + etc_certs=True, + read_only_etc_certs=True, + ) + env = [ + { + "name": "MOUNT_PATH", + "value": f"/work/{server.gl_project.path}", + }, + { + "name": "REPOSITORY_URL", + "value": server.gl_project.http_url_to_repo, + }, + { + "name": "LFS_AUTO_FETCH", + "value": "1" if server.server_options["lfs_auto_fetch"] else "0", + }, + {"name": "COMMIT_SHA", "value": server.commit_sha}, + {"name": "BRANCH", "value": server.branch}, + { + # used only for naming autosave branch + "name": "RENKU_USERNAME", + "value": server._user.username, + }, + { + "name": "GIT_AUTOSAVE", + "value": "1" if server.autosave_allowed else "0", + }, + { + "name": "GIT_URL", + "value": server._user.gitlab_client._base_url, + }, + { + "name": "GITLAB_OAUTH_TOKEN", + "value": server._user.git_token, + }, + ] + if type(server._user) is RegisteredUser: + env += [ + { + "name": "GIT_EMAIL", + "value": server._user.gitlab_user.email, + }, + { + "name": "GIT_FULL_NAME", + "value": server._user.gitlab_user.name, + }, + ] + return [ + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/initContainers/-", + "value": { + "image": current_app.config["GIT_CLONE_IMAGE"], + "name": "git-clone", + "resources": {}, + "securityContext": { + "allowPrivilegeEscalation": False, + "fsGroup": 100, + "runAsGroup": 100, + "runAsUser": 1000, + }, + "workingDir": "/", + "volumeMounts": [ + { + "mountPath": "/work", + "name": "workspace", + }, + *etc_cert_volume_mount, + ], + "env": env, + }, + }, + ], + } + ] + + +def certificates(): + initContainer = client.V1Container( + name="init-certificates", + image=current_app.config["CERTIFICATES_IMAGE"], + volume_mounts=get_certificates_volume_mounts( + etc_certs=True, + custom_certs=True, + read_only_etc_certs=False, + ), + ) + volume_etc_certs = client.V1Volume( + name="etc-ssl-certs", empty_dir=client.V1EmptyDirVolumeSource(medium="Memory") + ) + volume_custom_certs = client.V1Volume( + name="custom-ca-certs", + projected=client.V1ProjectedVolumeSource( + default_mode=440, + sources=[ + {"secret": {"name": i.get("secret")}} + for i in current_app.config["CUSTOM_CA_CERTS_SECRETS"] + if i is not None and i.get("secret") is not None + ], + ), + ) + api_client = client.ApiClient() + patches = [ + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/initContainers/-", + "value": api_client.sanitize_for_serialization(initContainer), + }, + ], + }, + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/volumes/-", + "value": api_client.sanitize_for_serialization(volume_etc_certs), + }, + ], + }, + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/volumes/-", + "value": api_client.sanitize_for_serialization(volume_custom_certs), + }, + ], + }, + ] + return patches diff --git a/renku_notebooks/api/amalthea_patches/inject_certificates.py b/renku_notebooks/api/amalthea_patches/inject_certificates.py new file mode 100644 index 000000000..9e6ab43cc --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/inject_certificates.py @@ -0,0 +1,52 @@ +from pathlib import Path + +from ..classes.user import RegisteredUser +from .utils import get_certificates_volume_mounts + + +def proxy(server): + etc_cert_volume_mounts = get_certificates_volume_mounts( + custom_certs=False, + etc_certs=True, + read_only_etc_certs=True, + ) + patches = [ + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": ( + "/statefulset/spec/template/spec/containers/1/volumeMounts/-" + ), + "value": volume_mount, + } + for volume_mount in etc_cert_volume_mounts + ], + }, + ] + if type(server._user) is RegisteredUser: + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/1/env/-", + "value": { + "name": "OAUTH2_PROXY_PROVIDER_CA_FILES", + "value": ",".join( + [ + ( + Path(volume_mount["mountPath"]) + / "ca-certificates.crt" + ).as_posix() + for volume_mount in etc_cert_volume_mounts + ] + ), + }, + }, + ], + }, + ) + return patches diff --git a/renku_notebooks/api/amalthea_patches/jupyter_server.py b/renku_notebooks/api/amalthea_patches/jupyter_server.py new file mode 100644 index 000000000..63d1a2d15 --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/jupyter_server.py @@ -0,0 +1,135 @@ +def env(server): + patches = [] + # amalthea always makes the jupyter server the first container in the statefulset + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/env/-", + "value": { + "name": "GIT_AUTOSAVE", + "value": "1" if server.autosave_allowed else "0", + }, + }, + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/env/-", + "value": { + "name": "RENKU_USERNAME", + "value": server._user.username, + }, + }, + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/env/-", + "value": {"name": "CI_COMMIT_SHA", "value": server.commit_sha}, + }, + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/env/-", + "value": { + "name": "NOTEBOOK_DIR", + "value": server.image_workdir.rstrip("/") + + f"/work/{server.gl_project.path}", + }, + }, + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/env/-", + # Note that inside the main container, the mount path is + # relative to $HOME. + "value": { + "name": "MOUNT_PATH", + "value": f"/work/{server.gl_project.path}", + }, + }, + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/env/-", + "value": {"name": "PROJECT_NAME", "value": server.project}, + }, + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/env/-", + "value": {"name": "GIT_CLONE_REPO", "value": "true"}, + }, + ], + } + ) + return patches + + +def args(): + patches = [] + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/containers/0/args", + "value": ["jupyter", "notebook"], + } + ], + } + ) + return patches + + +def image_pull_secret(server): + patches = [] + if server.is_image_private: + image_pull_secret_name = server.server_name + "-image-secret" + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/image_pull_secret", + "value": { + "apiVersion": "v1", + "data": { + ".dockerconfigjson": server._get_registry_secret() + }, + "kind": "Secret", + "metadata": { + "name": image_pull_secret_name, + "namespace": server._k8s_namespace, + }, + "type": "kubernetes.io/dockerconfigjson", + }, + } + ], + } + ) + patches.append( + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/imagePullSecrets/-", + "value": {"name": image_pull_secret_name}, + } + ], + } + ) + return patches + + +def disable_service_links(): + return [ + { + "type": "application/json-patch+json", + "patch": [ + { + "op": "add", + "path": "/statefulset/spec/template/spec/enableServiceLinks", + "value": False, + } + ], + } + ] diff --git a/renku_notebooks/api/amalthea_patches/s3mounts.py b/renku_notebooks/api/amalthea_patches/s3mounts.py new file mode 100644 index 000000000..e11754010 --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/s3mounts.py @@ -0,0 +1,8 @@ +def main(server): + s3mount_patches = [] + for i, s3mount in enumerate(server.s3mounts): + s3mount_name = f"{server.server_name}-ds-{i}" + s3mount_patches.append( + s3mount.get_manifest_patches(s3mount_name, server._k8s_namespace) + ) + return s3mount_patches diff --git a/renku_notebooks/api/amalthea_patches/utils.py b/renku_notebooks/api/amalthea_patches/utils.py new file mode 100644 index 000000000..5c17e346f --- /dev/null +++ b/renku_notebooks/api/amalthea_patches/utils.py @@ -0,0 +1,25 @@ +from flask import current_app +from kubernetes import client + + +def get_certificates_volume_mounts( + etc_certs=True, + custom_certs=True, + read_only_etc_certs=False, +): + volume_mounts = [] + etc_ssl_certs = client.V1VolumeMount( + name="etc-ssl-certs", + mount_path="/etc/ssl/certs/", + read_only=read_only_etc_certs, + ) + custom_ca_certs = client.V1VolumeMount( + name="custom-ca-certs", + mount_path=current_app.config["CUSTOM_CA_CERTS_PATH"], + read_only=True, + ) + if etc_certs: + volume_mounts.append(etc_ssl_certs) + if custom_certs: + volume_mounts.append(custom_ca_certs) + return client.ApiClient().sanitize_for_serialization(volume_mounts) diff --git a/renku_notebooks/api/classes/server.py b/renku_notebooks/api/classes/server.py index 3c0ceb798..f001737d4 100644 --- a/renku_notebooks/api/classes/server.py +++ b/renku_notebooks/api/classes/server.py @@ -1,5 +1,6 @@ from flask import current_app import gitlab +from itertools import chain from kubernetes import client from kubernetes.client.rest import ApiException from kubernetes.client.models import V1DeleteOptions @@ -8,6 +9,16 @@ from urllib.parse import urlparse, urljoin +from ..amalthea_patches import ( + autosave, + general, + git_proxy, + git_sidecar, + init_containers, + inject_certificates, + jupyter_server, + s3mounts as s3mounts_patches, +) from ...util.check_image import ( parse_image_name, get_docker_token, @@ -264,544 +275,31 @@ def _get_session_k8s_resources(self): } return resources - def _get_test_patches(self): - """RFC 6901 patches support test statements that will cause the whole patch - to fail if the test statements are not correct. This is used to ensure that the - order of containers in the amalthea manifests is what the notebook service expects.""" - patches = [] - container_names = ( - current_app.config["AMALTHEA_CONTAINER_ORDER_REGISTERED_SESSION"] - if type(self._user) is RegisteredUser - else current_app.config["AMALTHEA_CONTAINER_ORDER_ANONYMOUS_SESSION"] - ) - for container_ind, container_name in enumerate(container_names): - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "test", - "path": ( - "/statefulset/spec/template/spec" - f"/containers/{container_ind}/name" - ), - "value": container_name, - } - ], - } - ) - return patches - def _get_session_manifest(self): """Compose the body of the user session for the k8s operator""" - patches = self._get_test_patches() - prefix = current_app.config["RENKU_ANNOTATION_PREFIX"] - # Add labels and annotations - applied to overall manifest and secret only - labels = { - "app": "jupyter", - "component": "singleuser-server", - f"{prefix}commit-sha": self.commit_sha, - f"{prefix}gitlabProjectId": None, - f"{prefix}safe-username": self._user.safe_username, - f"{prefix}schemaVersion": current_app.config[ - "CURRENT_RESOURCE_SCHEMA_VERSION" - ], - } - annotations = { - f"{prefix}commit-sha": self.commit_sha, - f"{prefix}gitlabProjectId": None, - f"{prefix}safe-username": self._user.safe_username, - f"{prefix}username": self._user.username, - f"{prefix}servername": self.server_name, - f"{prefix}branch": self.branch, - f"{prefix}git-host": self.git_host, - f"{prefix}namespace": self.namespace, - f"{prefix}projectName": self.project, - f"{prefix}requested-image": self.image, - f"{prefix}repository": None, - } - if self.gl_project is not None: - labels[f"{prefix}gitlabProjectId"] = str(self.gl_project.id) - annotations[f"{prefix}gitlabProjectId"] = str(self.gl_project.id) - annotations[f"{prefix}repository"] = self.gl_project.web_url - # Add image pull secret if image is private - if self.is_image_private: - image_pull_secret_name = self.server_name + "-image-secret" - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/image_pull_secret", - "value": { - "apiVersion": "v1", - "data": { - ".dockerconfigjson": self._get_registry_secret() - }, - "kind": "Secret", - "metadata": { - "name": image_pull_secret_name, - "namespace": self._k8s_namespace, - "labels": labels, - "annotations": annotations, - }, - "type": "kubernetes.io/dockerconfigjson", - }, - } - ], - } - ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/imagePullSecrets/-", - "value": {"name": image_pull_secret_name}, - } - ], - } - ) - # Add git init / sidecar container - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/initContainers/-", - "value": { - "image": current_app.config["GIT_CLONE_IMAGE"], - "name": "git-clone", - "resources": {}, - "securityContext": { - "allowPrivilegeEscalation": False, - "fsGroup": 100, - "runAsGroup": 100, - "runAsUser": 1000, - }, - "workingDir": "/", - "volumeMounts": [ - { - "mountPath": "/work", - "name": "workspace", - } - ], - "env": [ - { - "name": "MOUNT_PATH", - "value": f"/work/{self.gl_project.path}", - }, - { - "name": "REPOSITORY_URL", - "value": self.gl_project.http_url_to_repo, - }, - { - "name": "LFS_AUTO_FETCH", - "value": "1" - if self.server_options["lfs_auto_fetch"] - else "0", - }, - {"name": "COMMIT_SHA", "value": self.commit_sha}, - {"name": "BRANCH", "value": self.branch}, - { - # used only for naming autosave branch - "name": "RENKU_USERNAME", - "value": self._user.username, - }, - { - "name": "GIT_AUTOSAVE", - "value": "1" if self.autosave_allowed else "0", - }, - { - "name": "GIT_URL", - "value": self._user.gitlab_client._base_url, - }, - { - "name": "GITLAB_OAUTH_TOKEN", - "value": self._user.git_token, - }, - ], - }, - } - ], - } - ) - # patches.append( - # { - # "type": "application/json-patch+json", - # "patch": [ - # { - # "op": "add", - # "path": "/statefulset/spec/template/spec/containers/-", - # "value": { - # "image": current_app.config["GIT_RPC_SERVER_IMAGE"], - # "name": "git-sidecar", - # # Do not expose this until access control is in place - # # "ports": [ - # # { - # # "containerPort": 4000, - # # "name": "git-port", - # # "protocol": "TCP", - # # } - # # ], - # "env": [ - # { - # "name": "MOUNT_PATH", - # "value": f"/work/{self.gl_project.path}", - # } - # ], - # "resources": {}, - # "securityContext": { - # "allowPrivilegeEscalation": False, - # "fsGroup": 100, - # "runAsGroup": 100, - # "runAsUser": 1000, - # }, - # "volumeMounts": [ - # { - # "mountPath": f"/work/{self.gl_project.path}/", - # "name": "workspace", - # "subPath": f"{self.gl_project.path}/", - # } - # ], - # # Enable readiness and liveness only when control is in place - # # "livenessProbe": { - # # "httpGet": {"port": 4000, "path": "/"}, - # # "periodSeconds": 30, - # # # delay should equal periodSeconds x failureThreshold - # # # from readiness probe values - # # "initialDelaySeconds": 600, - # # }, - # # the readiness probe will retry 36 times over 360 seconds to see - # # if the pod is ready to accept traffic - this gives the user session - # # a maximum of 360 seconds to setup the git sidecar and clone the repo - # # "readinessProbe": { - # # "httpGet": {"port": 4000, "path": "/"}, - # # "periodSeconds": 10, - # # "failureThreshold": 60, - # # }, - # }, - # } - # ], - # } - # ) - # Add git proxy container - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/-", - "value": { - "image": current_app.config["GIT_HTTPS_PROXY_IMAGE"], - "name": "git-proxy", - "env": [ - { - "name": "REPOSITORY_URL", - "value": self.gl_project.http_url_to_repo, - }, - {"name": "MITM_PROXY_PORT", "value": "8080"}, - {"name": "HEALTH_PORT", "value": "8081"}, - { - "name": "GITLAB_OAUTH_TOKEN", - "value": self._user.git_token, - }, - { - "name": "ANONYMOUS_SESSION", - "value": ( - "false" - if type(self._user) is RegisteredUser - else "true" - ), - }, - ], - "livenessProbe": { - "httpGet": {"path": "/health", "port": 8081}, - "initialDelaySeconds": 3, - }, - "readinessProbe": { - "httpGet": {"path": "/health", "port": 8081}, - "initialDelaySeconds": 3, - }, - }, - } - ], - } - ) - # add datashim secrets and datasets - s3mount_patches = [] - for i, s3mount in enumerate(self.s3mounts): - s3mount_name = f"{self.server_name}-ds-{i}" - s3mount_patches.append( - s3mount.get_manifest_patches(s3mount_name, self._k8s_namespace) - ) - patches += s3mount_patches - # disable service links that clutter env variable - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/enableServiceLinks", - "value": False, - } - ], - } - ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/volumes/-", - "value": { - "name": "notebook-helper-scripts-volume", - "configMap": { - "name": "notebook-helper-scripts", - "defaultMode": 493, - }, - }, - } - ], - } - ) - # We can add this to expose the git side-car once it's protected. - # patches.append( - # { - # "type": "application/json-patch+json", - # "patch": [ - # { - # "op": "add", - # "path": "/service/spec/ports/-", - # "value": { - # "name": "git-service", - # "port": 4000, - # "protocol": "TCP", - # "targetPort": 4000, - # }, - # } - # ], - # } - # ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/tolerations", - "value": current_app.config["SESSION_TOLERATIONS"], - } - ], - } - ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/affinity", - "value": current_app.config["SESSION_AFFINITY"], - } - ], - } - ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/nodeSelector", - "value": current_app.config["SESSION_NODE_SELECTOR"], - } - ], - } - ) - # amalthea always makes the jupyter server the first container in the statefulset - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/env/-", - "value": { - "name": "GIT_AUTOSAVE", - "value": "1" if self.autosave_allowed else "0", - }, - }, - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/env/-", - "value": { - "name": "RENKU_USERNAME", - "value": self._user.username, - }, - }, - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/env/-", - "value": {"name": "CI_COMMIT_SHA", "value": self.commit_sha}, - }, - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/env/-", - "value": { - "name": "NOTEBOOK_DIR", - "value": self.image_workdir.rstrip("/") - + f"/work/{self.gl_project.path}", - }, - }, - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/env/-", - # Note that inside the main container, the mount path is - # relative to $HOME. - "value": { - "name": "MOUNT_PATH", - "value": f"/work/{self.gl_project.path}", - }, - }, - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/env/-", - "value": {"name": "PROJECT_NAME", "value": self.project}, - }, - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/env/-", - "value": {"name": "GIT_CLONE_REPO", "value": "true"}, - }, - ], - } - ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/lifecycle", - "value": { - "preStop": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "/usr/local/bin/pre-stop.sh", - "||", - "true", - ] - } - } - }, - } - ], - } - ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/volumeMounts/-", - "value": { - "mountPath": "/usr/local/bin/pre-stop.sh", - "name": "notebook-helper-scripts-volume", - "subPath": "pre-stop.sh", - }, - } - ], - } - ) - # patches.append( - # { - # "type": "application/json-patch+json", - # "patch": [ - # { - # "op": "add", - # "path": "/statefulset/spec/template/spec/containers/0/command", - # "value": ["bash", "-c"], - # } - # ], - # } - # ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/0/args", - "value": ["jupyter", "notebook"], - } - ], - } - ) - if type(self._user) is RegisteredUser: - # modify oauth2 proxy for dev purposes only - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - "path": "/statefulset/spec/template/spec/containers/1/env/-", - "value": { - "name": "OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL", - "value": current_app.config[ - "OIDC_ALLOW_UNVERIFIED_EMAIL" - ], - }, - }, - { - "op": "add", - "path": "/statefulset/spec/template/spec/initContainers/0/env/-", - "value": { - "name": "GIT_EMAIL", - "value": self._user.gitlab_user.email, - }, - }, - { - "op": "add", - "path": "/statefulset/spec/template/spec/initContainers/0/env/-", - "value": { - "name": "GIT_FULL_NAME", - "value": self._user.gitlab_user.name, - }, - }, - ], - } + patches = list( + chain( + general.test(self), + general.session_tolerations(), + general.session_affinity(), + general.session_node_selector(), + jupyter_server.args(), + jupyter_server.env(self), + jupyter_server.image_pull_secret(self), + jupyter_server.disable_service_links(), + autosave.main(), + git_proxy.main(self), + git_sidecar.main(), + general.oidc_unverified_email(self), + s3mounts_patches.main(self), + # init container for certs must come before all other init containers + # so that it runs first before all other init containers + init_containers.certificates(), + init_containers.git_clone(self), + inject_certificates.proxy(self), ) - patches.append( - { - "type": "application/json-patch+json", - "patch": [ - { - "op": "add", - # "~1" == "/" for rfc6902 json patches - "path": ( - "/ingress/metadata/annotations/" - "nginx.ingress.kubernetes.io~1configuration-snippet" - ), - "value": ( - 'more_set_headers "Content-Security-Policy: ' - "frame-ancestors 'self' " - f'{self.server_url}";' - ), - } - ], - } ) + # Storage if current_app.config["NOTEBOOKS_SESSION_PVS_ENABLED"]: storage = { "size": self.server_options["disk_request"], @@ -823,6 +321,7 @@ def _get_session_manifest(self): "mountPath": self.image_workdir.rstrip("/") + "/work", }, } + # Authentication if type(self._user) is RegisteredUser: session_auth = { "token": "", @@ -841,13 +340,14 @@ def _get_session_manifest(self): "token": self._user.username, "oidc": {"enabled": False}, } + # Combine everything into the manifest manifest = { "apiVersion": f"{current_app.config['CRD_GROUP']}/{current_app.config['CRD_VERSION']}", "kind": "JupyterServer", "metadata": { "name": self.server_name, - "labels": labels, - "annotations": annotations, + "labels": self.get_labels(), + "annotations": self.get_annotations(), }, "spec": { "auth": session_auth, @@ -1096,3 +596,36 @@ def _get_server_options_from_js(js): **current_app.config["SERVER_OPTIONS_DEFAULTS"], **server_options, } + + def get_annotations(self): + prefix = current_app.config["RENKU_ANNOTATION_PREFIX"] + annotations = { + f"{prefix}commit-sha": self.commit_sha, + f"{prefix}gitlabProjectId": None, + f"{prefix}safe-username": self._user.safe_username, + f"{prefix}username": self._user.username, + f"{prefix}servername": self.server_name, + f"{prefix}branch": self.branch, + f"{prefix}git-host": self.git_host, + f"{prefix}namespace": self.namespace, + f"{prefix}projectName": self.project, + f"{prefix}requested-image": self.image, + f"{prefix}repository": None, + } + if self.gl_project is not None: + annotations[f"{prefix}gitlabProjectId"] = str(self.gl_project.id) + annotations[f"{prefix}repository"] = self.gl_project.web_url + return annotations + + def get_labels(self): + prefix = current_app.config["RENKU_ANNOTATION_PREFIX"] + labels = { + "app": "jupyter", + "component": "singleuser-server", + f"{prefix}commit-sha": self.commit_sha, + f"{prefix}gitlabProjectId": None, + f"{prefix}safe-username": self._user.safe_username, + } + if self.gl_project is not None: + labels[f"{prefix}gitlabProjectId"] = str(self.gl_project.id) + return labels diff --git a/renku_notebooks/config.py b/renku_notebooks/config.py index 8ee0176ee..67dd86ddf 100644 --- a/renku_notebooks/config.py +++ b/renku_notebooks/config.py @@ -85,6 +85,9 @@ OIDC_TOKEN_URL = os.environ.get("OIDC_TOKEN_URL") OIDC_AUTH_URL = os.environ.get("OIDC_AUTH_URL") OIDC_ALLOW_UNVERIFIED_EMAIL = os.environ.get("OIDC_ALLOW_UNVERIFIED_EMAIL") +CUSTOM_CA_CERTS_PATH = "/usr/local/share/ca-certificates" +CERTIFICATES_IMAGE = os.environ.get("CERTIFICATES_IMAGE") +CUSTOM_CA_CERTS_SECRETS = safe_load(os.environ.get("CUSTOM_CA_CERTS_SECRETS", "[]")) CRD_GROUP = os.environ.get("CRD_GROUP") CRD_VERSION = os.environ.get("CRD_VERSION")