Refactor image build check

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:8.11.1 as react-builder
+FROM node:8.11.1-alpine as react-builder
 
 # Uncomment the following line to allow live updating of files for development.
 # ADD . /app

helm-chart/chartpress.yaml

@@ -24,7 +24,7 @@ charts:
         contextPath: ../singleuser
         buildArgs:
           JUPYTERHUB_VERSION: "0.9.2"
-          RENKU_VERSION: "0.2.0rc1"
+          RENKU_VERSION: "0.2.0"
         dockerfilePath: ../singleuser/Dockerfile
         valuesPath: jupyterhub.singleuser.image
         paths:

helm-chart/renku-notebooks/templates/deployment.yaml

@@ -28,26 +28,36 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
+            - name: DEFAULT_NOTEBOOK_IMAGE
+              value: "{{ .Values.jupyterhub.singleuser.image.name }}:{{ .Values.jupyterhub.singleuser.image.tag }}"
             {{ if eq .Values.debug true }}
             - name: FLASK_DEBUG
               value: "1"
             {{ end }}
             - name: JUPYTERHUB_API_TOKEN
-              value: {{ .Values.jupyterhub_api_token }}
+              value: {{ .Values.jupyterhub.hub.services.notebooks.api_token }}
             - name: JUPYTERHUB_API_URL
-            {{ if .Values.jupyterhub_api_url }}
-              value: {{ .Values.jupyterhub_api_url }}
-            {{ else }}
-              value: {{ template "notebooks.http" . }}://{{ .Values.global.renku.domain }}{{ .Values.jupyterhub_base_url }}hub/api
-            {{ end }}
+              value: {{ template "notebooks.http" . }}://{{ .Values.global.renku.domain }}{{ .Values.jupyterhub.hub.baseUrl }}hub/api
+            - name: JUPYTERHUB_SINGLEUSER_DEFAULT_URL
+              value: {{ .Values.jupyterhub.singleuser.defaultUrl }}
             - name: JUPYTERHUB_SERVICE_PREFIX
-              value: {{ .Values.jupyterhub_base_url }}services/notebooks/
+              value: {{ .Values.jupyterhub.hub.baseUrl }}services/notebooks/
             - name: JUPYTERHUB_BASE_URL
-              value: {{ .Values.jupyterhub_base_url }}
+              value: {{ .Values.jupyterhub.hub.baseUrl }}
             - name: JUPYTERHUB_CLIENT_ID
               value: {{ .Values.jupyterhub.hub.services.notebooks.oauth_client_id }}
             - name: GITLAB_REGISTRY_SECRET
               value: {{ template "renku.fullname" . }}-registry
+            - name: GITLAB_URL
+            {{ if .Values.gitlab.url }}
+              value: {{ .Values.gitlab.url }}
+            {{ else }}
+              value: {{ template "notebooks.http" . }}://{{ .Values.global.renku.domain}}{{ .Values.global.gitlab.urlPrefix }}
+            {{ end }}
+            - name: IMAGE_BUILD_TIMEOUT
+              value: "{{ .Values.imageBuildTimeout }}"
+            - name: IMAGE_REGISTRY
+              value: {{required "An image registry must be specified." .Values.imageRegistry }}
           ports:
             - name: http
               containerPort: 8000

helm-chart/renku-notebooks/values.yaml

@@ -5,6 +5,10 @@
 global:
   useHTTPS: false
 
+gitlab:
+  ## specify the GitLab instance URL
+  url:
+
 replicaCount: 1
 
 image:
@@ -24,6 +28,12 @@ image:
 # jupyterhub_base_url: /
 # jupyterhub_api_url: http://<host>/hub/api
 
+## Set the default image registry
+# imageRegistry:
+
+## set the image build timeout
+imageBuildTimeout: '600'
+
 # enable the security context - if using telepresence for development, disable it
 securityContext:
   enabled: true
@@ -89,8 +99,6 @@ jupyterhub:
     extraEnv:
       DEBUG: 1
       JUPYTERHUB_SPAWNER_CLASS: spawners.RenkuKubeSpawner
-      ## timeout for waiting on an image build
-      # GITLAB_IMAGE_BUILD_TIMEOUT: 600
       ## GitLab instance URL
       # GITLAB_URL: http://gitlab.com
     extraConfig: |

jupyterhub/jupyterhub-k8s.Dockerfile

@@ -1,9 +1,9 @@
-FROM jupyterhub/k8s-hub:bf3550b
+FROM jupyterhub/k8s-hub:0.7.0
 
 USER root
 
-COPY requirements-k8s.txt /tmp/requirements.renku.txt
-RUN pip3 install -r /tmp/requirements.renku.txt --no-cache-dir
+COPY requirements-k8s.txt /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt --no-cache-dir
 
 COPY spawners.py /usr/local/lib/python3.6/dist-packages/
 

jupyterhub/spawners.py

@@ -89,18 +89,16 @@ def start(self, *args, **kwargs):
         auth_state = yield self.user.get_auth_state()
         assert 'access_token' in auth_state
 
-        # 1. check authorization against GitLab
         options = self.user_options
         namespace = options.get('namespace')
         project = options.get('project')
         commit_sha = options.get('commit_sha')
         commit_sha_7 = commit_sha[:7]
+        self.image = options.get('image')
 
         url = os.getenv('GITLAB_URL', 'http://gitlab.renku.build')
 
-        # image build timeout -- configurable, defaults to 10 minutes
-        image_build_timeout = int(os.getenv('GITLAB_IMAGE_BUILD_TIMEOUT', 600))
-
+        # check authorization against GitLab
         gl = gitlab.Gitlab(
             url, api_version=4, oauth_token=auth_state['access_token']
         )
@@ -109,7 +107,6 @@ def start(self, *args, **kwargs):
             gl.auth()
             gl_project = gl.projects.get('{0}/{1}'.format(namespace, project))
             self.gl_user = gl.user
-            self.log.info('Got user profile: {}'.format(self.gl_user))
 
             # gather project permissions for the logged in user
             permissions = gl_project.attributes['permissions']
@@ -135,57 +132,7 @@ def start(self, *args, **kwargs):
             raise web.HTTPError(401, 'Not authorized to view project.')
             return
 
-        # set default image
-        self.image = os.getenv(
-            'JUPYTERHUB_NOTEBOOK_IMAGE', 'renku/singleuser:latest'
-        )
 
-        for pipeline in gl_project.pipelines.list():
-            if pipeline.attributes['sha'] == commit_sha:
-                status = self._get_job_status(pipeline, 'image_build')
-
-                if not status:
-                    # there is no image_build job for this commit
-                    # so we use the default image
-                    self.log.info('No image_build job found in pipeline.')
-                    break
-
-                # we have an image_build job in the pipeline, check status
-                timein = time.time()
-                # TODO: remove this loop altogether and only request the launch
-                # when the image is ready
-                while time.time() - timein < image_build_timeout:
-                    if status == 'success':
-                        # the image was built
-                        # it *should* be there so lets use it
-                        self.image = '{image_registry}'\
-                                '/{namespace}'\
-                                '/{project}'\
-                                ':{commit_sha_7}'.format(
-                                    image_registry=os.getenv('IMAGE_REGISTRY'),
-                                    commit_sha_7=commit_sha_7,
-                                    **options
-                        ).lower()
-                        self.log.info(
-                            'Using image {image}.'.format(image=self.image)
-                        )
-                        break
-                    elif status in {'failed', 'canceled'}:
-                        self.log.info(
-                            'Image build failed for project {0} commit {1} - '
-                            'using {2} instead'.format(
-                                project, commit_sha, self.image
-                            )
-                        )
-                        break
-                    yield gen.sleep(5)
-                    status = self._get_job_status(pipeline, 'image_build')
-                    self.log.debug(
-                        'status of image_build job for commit '
-                        '{commit_sha_7}: {status}'.format(
-                            commit_sha_7=commit_sha_7, status=status
-                        )
-                    )
 
         self.cmd = 'jupyterhub-singleuser'
         try:
@@ -203,16 +150,6 @@ def start(self, *args, **kwargs):
 
         return result
 
-    @staticmethod
-    def _get_job_status(pipeline, job_name):
-        """Helper method to retrieve job status based on the job name."""
-        status = [
-            job.attributes['status'] for job in pipeline.jobs.list()
-            if job.attributes['name'] == job_name
-        ]
-        return status.pop() if status else None
-
-
 try:
     import docker
     from dockerspawner import DockerSpawner
@@ -354,6 +291,9 @@ def get_pod_manifest(self):
             # set the notebook container image
             self.image_spec = self.image
 
+            # set the default url
+            self.default_url = options.get('default_url')
+
             #: Define a new empty volume.
             self.volumes = [
                 volume for volume in self.volumes if volume['name'] != volume_name

singleuser/Dockerfile

@@ -1,12 +1,12 @@
 # Build as renku/singleuser
 # Run with the DockerSpawner in JupyterHub
 
-ARG JUPYTERHUB_VERSION=0.9
+ARG JUPYTERHUB_VERSION=0.9.2
 ARG BASE_IMAGE=jupyterhub/singleuser:$JUPYTERHUB_VERSION
 
 FROM $BASE_IMAGE
 ARG JUPYTERHUB_VERSION
-ARG RENKU_VERSION=0.2.0rc1
+ARG RENKU_VERSION=0.2.0
 MAINTAINER Swiss Data Science Center <info@datascience.ch>
 
 
@@ -33,8 +33,10 @@ RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.d
 USER $NB_USER
 
 # install jupyterlab, papermill, git extension and renku-jupyterlab-ts
+COPY requirements.txt /tmp/requirements.txt
+
 RUN python3 -m pip install -U pip && \
-    pip install -U jupyterlab papermill && \
+    pip install -r /tmp/requirements.txt && \
     jupyter labextension update @jupyterlab/hub-extension --no-build && \
     jupyter labextension install @jupyterlab/git --no-build && \
     jupyter labextension install renku-jupyterlab-ts --no-build && \

singleuser/requirements.txt

@@ -0,0 +1,2 @@
+jupyterlab==0.34.0
+papermill