enable jupyterhub, jupyterlab, dashserve (#60)

updates

Author: miraculixx

Procfile

@@ -1,3 +1,3 @@
 worker: celery worker --app omegaml.celeryapp -E -B --loglevel=DEBUG
-notebook: jupyter notebook --allow-root --ip 0.0.0.0 --no-browser $JUPYTER_PARAM
+notebook: jupyter notebook --allow-root --ip 0.0.0.0 --no-browser $JUPYTER_PARAM --port $JUPYTER_PORT
 restapi: python -m omegaml.restapi

config-example.yml

@@ -0,0 +1,5 @@
+# example user config file
+#OMEGA_USER_EXTENSIONS:
+#    OMEGA_STORE_BACKENDS:
+#         anything: omegaml.example.userext.mymodule.MyBackend
+

docker-compose.yml

@@ -5,19 +5,47 @@ services:
        image: omegaml/omegaml:latest
        hostname: omegaml
        ports:
-          - "8899:8888"
           - "5000:5000"
+          - "8899:8899"
        links:
           - rabbitmq
           - mongodb
        working_dir: /app
-       command: honcho start notebook restapi
+       command: honcho start restapi notebook
        environment:
          - APP=omegaml
          - OMEGA_MONGO_URL=mongodb://mongodb/omega
          - OMEGA_BROKER=amqp://rabbitmq:5672//
          - OMEGA_FRAMEWORKS=scikit-learn,keras,tensorflow
          - JUPYTER_PASSWORD=sha1:24fa20fec60f:c7cd7e46afa507d484c59abeadbefa05022583b8
+         - JUPYTER_PORT=8899
+         - PYTHONPATH=/app/pylib
+       volumes:
+         - pythonlib:/app/pylib
+   jyhub:
+       build:
+         context: scripts/docker/jyhub
+         args:
+           - pypi
+       image: omegaml/jyhub:latest
+       hostname: jyhub
+       ports:
+          - "8000:8000"
+          - "8050-8060:8050-8060"
+       links:
+          - omegaml
+          - rabbitmq
+          - mongodb
+       command: jupyterhub
+       environment:
+         - APP=omegaml
+         - OMEGA_MONGO_URL=mongodb://mongodb/omega
+         - OMEGA_BROKER=amqp://rabbitmq:5672//
+         - OMEGA_FRAMEWORKS=scikit-learn,keras,tensorflow
+         - JUPYTER_PASSWORD=sha1:24fa20fec60f:c7cd7e46afa507d484c59abeadbefa05022583b8
+         - PYTHONPATH=/app/pylib
+       volumes:
+         - pythonlib:/app/pylib
    worker:
        image: omegaml/omegaml:latest
        hostname: worker
@@ -30,10 +58,18 @@ services:
          - OMEGA_MONGO_URL=mongodb://mongodb/omega
          - OMEGA_BROKER=amqp://rabbitmq:5672//
          - OMEGA_FRAMEWORKS=scikit-learn,keras,tensorflow
+         - JY_CONTENTS_MANAGER=
          - C_FORCE_ROOT=yes
    rabbitmq:
        image: rabbitmq:3.7.17
        hostname: rabbitmq
+       ports:
+          - 5672:5672
    mongodb:
        image: mongo:3.6.8-stretch
        hostname: mongodb
+       ports:
+          - 27017:27017
+
+volumes:
+   pythonlib:

omegaml/client/cli/jobs.py

@@ -145,7 +145,8 @@ def schedule(self):
                 self.logger.info("Given this new interval, next {next_n} times would be:".format(**locals()))
                 for time in om.jobs.Schedule.from_cron(cron_sched).next_times(int(next_n)):
                     self.logger.info("  {}".format(time))
-            answer = self.ask("Do you want to schedule {name} at {human_sched}?", options="Y/n", default='y')
+            text = "Do you want to schedule {name} at {human_sched}?".format(**locals())
+            answer = self.ask(text, options="Y/n", default='y')
             if answer.lower().startswith('n'):
                 self.logger.info('Ok, not scheduled. Try again.')
                 return

omegaml/documents.py

@@ -9,6 +9,7 @@
 
 from omegaml.util import settings
 
+
 # default kinds of objects
 class MDREGISTRY:
     PANDAS_DFROWS = 'pandas.dfrows'  # dataframe
@@ -63,6 +64,8 @@ class Metadata(Document):
         uri = StringField()
         #: created datetime
         created = DateTimeField(default=datetime.datetime.now)
+        #: created datetime
+        modified = DateTimeField(default=datetime.datetime.now)
         # the actual db is defined in settings, OMEGA_MONGO_URL
         meta = {
             'db_alias': 'omega',
@@ -84,6 +87,10 @@ def __unicode__(self):
                       for k in self._fields.keys() if k in fields)
             return u"Metadata(%s)" % ','.join(kwargs)
 
+        def save(self, *args, **kwargs):
+            self.modified = datetime.datetime.now()
+            return super().save(*args, **kwargs)
+
     return Metadata
 
 

omegaml/notebook/jupyter/jupyter_notebook_config.py

@@ -621,7 +621,7 @@ def setup():
 
 # omegaml setup
 default_contents_manager = 'omegaml.notebook.omegacontentsmgr.OmegaStoreContentsManager'
-contents_manager = os.environ.get('JY_CONTENTS_MANAGER', default_contents_manager)
+contents_manager = os.environ.get('JY_CONTENTS_MANAGER') or default_contents_manager
 c.NotebookApp.contents_manager_class = contents_manager
 
 import logging

omegaml/notebook/omegacontentsmgr.py

@@ -2,11 +2,13 @@
 from urllib.parse import unquote
 
 import nbformat
+from datetime import datetime
 from notebook.services.contents.manager import ContentsManager
 from tornado import web
 
 from omegaml.notebook.checkpoints import NoOpCheckpoints
 
+
 class OmegaStoreContentsManager(ContentsManager):
     """
     Jupyter notebook storage manager for omegaml
@@ -124,7 +126,7 @@ def delete_file(self, path):
 
     def rename_file(self, old_path, new_path):
         """
-        rename a file 
+        rename a file
 
         this is called by the contents engine to rename an entry
         """
@@ -136,8 +138,9 @@ def rename_file(self, old_path, new_path):
         elif self.dir_exists(new_path):
             raise web.HTTPError(409, u'Directory already exists: %s' % new_path)
         # do the renaming
-        if self.dir_exists(old_path):
-            meta = self.omega.jobs.metadata(old_path + '/' + self._dir_placeholder)
+        dirname = old_path + '/' + self._dir_placeholder
+        if self.dir_exists(dirname):
+            meta = self.omega.jobs.metadata(dirname)
             meta.name = new_path + '/' + self._dir_placeholder
         elif self.file_exists(old_path):
             meta = self.omega.jobs.metadata(old_path)
@@ -150,7 +153,7 @@ def exists(self, path):
         Does a file or dir exist at the given collection in gridFS?
         We do not have dir so dir_exists returns true.
 
-        :param path: (str) The relative path to the file's directory 
+        :param path: (str) The relative path to the file's directory
           (with '/' as separator)
         :returns exists: (boo) The relative path to the file's directory (with '/' as separator)
         """
@@ -190,22 +193,22 @@ def _notebook_model(self, path, content=True):
             model['content'] = nb
             model['format'] = 'json'
             self.validate_notebook_model(model)
-        # if exists already fake last modified and created timestamps
-        # otherwise jupyter notebook will claim a newer version "on disk"
-        if self.exists(path):
-            from IPython.utils import tz
-            # FIXME get actual datetime
-            model['last_modified'] = tz.datetime(1970, 1, 1)
-            model['created'] = tz.datetime(1970, 1, 1)
+        # always add accurate created and modified
+        meta = self.omega.jobs.metadata(path)
+        if meta is not None:
+            model['created'] = meta.created
+            model['last_modified'] = meta.modified
+        else:
+            model['last_modified'] = datetime.utcnow()
+            model['created'] = datetime.utcnow()
         return model
 
     def _base_model(self, path, kind=None):
         """Build the common base of a contents model"""
         # http://jupyter-notebook.readthedocs.io/en/stable/extending/contents.html
-        from IPython.utils import tz
         path = unquote(path).strip('/')
-        last_modified = tz.utcnow()
-        created = tz.utcnow()
+        last_modified = datetime.utcnow()
+        created = last_modified
         # Create the base model.
         model = {}
         model['name'] = os.path.basename(path)

omegaml/notebook/tasks.py

@@ -86,10 +86,9 @@ def execute_scripts(self, **kwargs):
         # run pending jobs
         pending = (trigger for trigger in triggers
                    if trigger['event-kind'] == 'scheduled' and trigger['status'] == 'PENDING')
-        logger.debug("***** {}".format(pending))
         for trigger in pending:
             run_at = trigger['run-at']
-            logger.info("***** {} {}".format(now, run_at))
+            logger.info("***** now={} run_at={}".format(now, run_at))
             if now >= run_at:
                 om.runtime.job(job_meta.name).run(event=trigger['event'])
                 # immediately schedule for next time

omegaml/restapi/__main__.py

@@ -1,4 +1,7 @@
+import os
+
 from omegaml.restapi.app import app
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=5000, debug=True)
+    port = os.environ.get('OMEGA_PORT')
+    app.run(host='0.0.0.0', port=port, debug=True)

omegaml/tests/features/steps/notebook.py

@@ -10,10 +10,16 @@ def open_jupyter(ctx):
     br = ctx.browser
     br.visit(ctx.feature.jynb_url)
     nb = Notebook(br)
-    login_required = br.is_text_present('Password', wait_time=2)
-    login_required |= br.is_text_present('token', wait_time=2)
-    if login_required:
-        nb.login()
+    if br.is_text_present('JupyterHub', wait_time=10):
+        login_required = br.is_element_present_by_id('username_input', wait_time=30)
+        if login_required:
+            nb.login_hub()
+    else:
+        # fallback to juypter notebook
+        login_required = br.is_text_present('Password', wait_time=2)
+        login_required |= br.is_text_present('token', wait_time=2)
+        if login_required:
+            nb.login_nb()
     nb.jupyter_home
 
 
@@ -57,7 +63,7 @@ def list_datasets(ctx):
     sleep(10)
     current = nb.current_cell_output()
     expected = "['sample']"
-    assert current == expected, "Expected {expected}, got {current}".format(**locals)
+    assert current == expected, "Expected {expected}, got {current}".format(**locals())
 
 
 @then(u'we can add a notebook in the folder')