From 932fbf24918e04d48d3c61e2ddd4d7160764df66 Mon Sep 17 00:00:00 2001 From: Scott Barnett Date: Sun, 15 Nov 2020 21:00:09 +1100 Subject: [PATCH 1/2] ENHANCE: Add Docker file for Jupyter support --- surround/project.py | 4 +- templates/new/Dockerfile.Notebook.txt | 8 ++ templates/new/data_analysis.ipynb.txt | 56 +++++++++++ templates/new/dodo.py.txt | 129 +++++--------------------- templates/new/example.ipynb.txt | 35 ------- 5 files changed, 90 insertions(+), 142 deletions(-) create mode 100644 templates/new/Dockerfile.Notebook.txt create mode 100644 templates/new/data_analysis.ipynb.txt delete mode 100644 templates/new/example.ipynb.txt diff --git a/surround/project.py b/surround/project.py index d5ee448f..7d99fc23 100644 --- a/surround/project.py +++ b/surround/project.py @@ -27,13 +27,13 @@ ("{project_name}/__main__.py", "batch_main.py.txt", False, False), ("{project_name}/__main__.py", "web_main.py.txt", False, True), ("{project_name}/__init__.py", "init.py.txt", False, False), - ("notebooks/example.ipynb", "example.ipynb.txt", False, False), - ("notebooks/example.ipynb", "example.ipynb.txt", False, True), + ("notebooks/data_analysis.ipynb", "data_analysis.ipynb.txt", False, False), ("templates/results.html", "results.html.txt", False, False), ("templates/results.html", "results.html.txt", False, True), ("dodo.py", "dodo.py.txt", False, False), ("dodo.py", "web_dodo.py.txt", False, True), ("Dockerfile", "Dockerfile.txt", False, False), + ("Dockerfile.Notebook", "Dockerfile.Notebook.txt", False, False), ("{project_name}/config.yaml", "config.yaml.txt", False, False), (".gitignore", ".gitignore.txt", False, False) ] diff --git a/templates/new/Dockerfile.Notebook.txt b/templates/new/Dockerfile.Notebook.txt new file mode 100644 index 00000000..ec60c760 --- /dev/null +++ b/templates/new/Dockerfile.Notebook.txt @@ -0,0 +1,8 @@ +FROM a2i2/{project_name}:latest + +RUN pip3 install jupyter -U && pip3 install jupyterlab + +EXPOSE 8888 + +# TODO: Remove token and password settings for production deployment +CMD ["jupyter", "lab","--allow-root", "--ip=0.0.0.0", "--no-browser","--NotebookApp.token=''","--NotebookApp.password=''"] diff --git a/templates/new/data_analysis.ipynb.txt b/templates/new/data_analysis.ipynb.txt new file mode 100644 index 00000000..1f4ccd67 --- /dev/null +++ b/templates/new/data_analysis.ipynb.txt @@ -0,0 +1,56 @@ +{{ + "cells": [ + {{ + "cell_type": "code", + "execution_count": 1, + "metadata": {{}}, + "outputs": [], + "source": [ + "# Magic commands\n", + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "# Set up paths to access project module\n", + "import sys\n", + "import os\n", + "sys.path.append('/app')\n", + "os.chdir('/app')" + ] + }}, + {{ + "cell_type": "code", + "execution_count": 2, + "metadata": {{}}, + "outputs": [], + "source": [ + "# Package imports\n", + "from surround import Config\n", + "from {project_name}.file_system_runner import FileSystemRunner\n", + "\n", + "config = Config(auto_load=True)\n", + "data = FileSystemRunner().load_data(None, config)" + ] + }} + ], + "metadata": {{ + "kernelspec": {{ + "display_name": "Python 3", + "language": "python", + "name": "python3" + }}, + "language_info": {{ + "codemirror_mode": {{ + "name": "ipython", + "version": 3 + }}, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + }} + }}, + "nbformat": 4, + "nbformat_minor": 4 +}} diff --git a/templates/new/dodo.py.txt b/templates/new/dodo.py.txt index 4dbb6930..07999b35 100644 --- a/templates/new/dodo.py.txt +++ b/templates/new/dodo.py.txt @@ -5,9 +5,6 @@ This module defines the tasks that can be executed using `surround run [task nam import os import sys import subprocess -import re -import webbrowser -import logging from pathlib import Path @@ -19,7 +16,8 @@ CONFIG = Config(os.path.dirname(__file__)) DOIT_CONFIG = {{'verbosity':2, 'backend':'sqlite3'}} PACKAGE_PATH = os.path.basename(CONFIG["package_path"]) IMAGE = "%s/%s:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"]) -LOGGER = logging.getLogger(__name__) +IMAGE_JUPYTER = "%s/%s-jupyter:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"]) +DOCKER_JUPYTER = "Dockerfile.Notebook" PARAMS = [ {{ @@ -46,7 +44,7 @@ def task_build(): def task_remove(): """Remove the Docker image for the current project""" return {{ - 'actions': ['docker rmi %s -f' % IMAGE], + 'actions': ['docker rmi %s %s -f' % (IMAGE, IMAGE_JUPYTER)], 'params': PARAMS }} @@ -199,107 +197,28 @@ def task_batch_local(): 'params': PARAMS }} +def task_build_jupyter(): + """Build the Docker image for a Jupyter Lab notebook""" + return {{ + 'basename': 'buildJupyter', + 'actions': ['docker build --tag=%s . -f %s' % (IMAGE_JUPYTER, DOCKER_JUPYTER)], + 'task_dep': ['build'], + 'params': PARAMS + }} + def task_jupyter(): - """Run a Jupyter notebook in the Docker container""" - # Allow for auto reload to be enabled and import modules from project package - ipython_config = "c.InteractiveShellApp.extensions.append('autoreload')\n" - ipython_config += "c.InteractiveShellApp.exec_lines = " - ipython_config += "['%autoreload 2', 'import sys', 'sys.path.append(\\'../\\')']" - - # Build the command for running jupyter - command = [ - "pip install -r /app/requirements.txt", - "mkdir /etc/ipython", - "echo \"%s\" > /etc/ipython/ipython_config.py" % ipython_config, - "/usr/local/bin/start.sh jupyter notebook --NotebookApp.token=''" + """Run a Jupyter Lab notebook""" + cmd = [ + "docker", + "run", + "-itp", + "8888:8888", + '-w', + '/app', + "--volume", + "\"%s/\":/app" % CONFIG["volume_path"], + IMAGE_JUPYTER ] - command = "; ".join(command) - - def run_command(): - process = subprocess.Popen( - [ - "docker", - "run", - "--rm", - "--name", - "{project_name}_surround_notebook", - "--volume", - "%s:/app" % CONFIG['volume_path'], - "-p", - "55910:8888", - "--user", - "root", - "-w", - "/app", - "jupyter/base-notebook:307ad2bb5fce", - "bash", - "-c", - command - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8') - - LOGGER.info("Starting jupyter notbook server...\n") - - # Get the IP address of the container, otherwise use localhost - ip_process = subprocess.Popen( - ['docker-machine', 'ip'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8') - ip_process.wait() - - ip_output = ip_process.stdout.readline().rstrip() - - if re.match(r"^(\d{{1,3}}\.){{3}}\d{{1,3}}$", ip_output): - host = ip_output - else: - host = "localhost" - - # Wait for the notebook server to be up before loading browser - while True: - line = process.stderr.readline().rstrip() - if line and 'Serving notebooks from local directory' in line: - break - - if process.poll(): - LOGGER.error("Failed to start the server, check if its not running somewhere else!") - - # Stop any containers that might be running - process = subprocess.Popen( - [ - 'docker', - 'stop', - '{project_name}_surround_notebook' - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait() - return - - # Open the browser automatically - webbrowser.open('http://%s:55910/tree' % host, new=2) - - LOGGER.info("Notebook URL: http://%s:55910/tree\n", host) - LOGGER.info("Use CTRL+C to stop the server.") - - try: - process.wait() - except KeyboardInterrupt: - pass - finally: - LOGGER.info("Closing server...") - process = subprocess.Popen( - [ - 'docker', - 'stop', - '{project_name}_surround_notebook' - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait() - return {{ - 'actions': [run_command] + 'actions': [" ".join(cmd)], }} diff --git a/templates/new/example.ipynb.txt b/templates/new/example.ipynb.txt deleted file mode 100644 index f3745d41..00000000 --- a/templates/new/example.ipynb.txt +++ /dev/null @@ -1,35 +0,0 @@ -{{ - "cells": [ - {{ - "cell_type": "code", - "execution_count": 7, - "metadata": {{}}, - "outputs": [], - "source": [ - "# Example importing code from the Surround project\n", - "from {project_name}.stages import Main" - ] - }} - ], - "metadata": {{ - "kernelspec": {{ - "display_name": "Python 3", - "language": "python", - "name": "python3" - }}, - "language_info": {{ - "codemirror_mode": {{ - "name": "ipython", - "version": 3 - }}, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - }} - }}, - "nbformat": 4, - "nbformat_minor": 2 -}} From e4cb341c34048b581a9e09a0cd468f2a8a4e54b7 Mon Sep 17 00:00:00 2001 From: Scott Barnett Date: Thu, 26 Nov 2020 11:43:59 +1100 Subject: [PATCH 2/2] FIX: Add Jupyter update to the web template --- templates/new/web_dodo.py.txt | 126 ++++++---------------------------- 1 file changed, 22 insertions(+), 104 deletions(-) diff --git a/templates/new/web_dodo.py.txt b/templates/new/web_dodo.py.txt index 8e346cd1..a54eeda5 100644 --- a/templates/new/web_dodo.py.txt +++ b/templates/new/web_dodo.py.txt @@ -5,9 +5,6 @@ This module defines the tasks that can be executed using `surround run [task nam import os import sys import subprocess -import re -import webbrowser -import logging from pathlib import Path @@ -19,7 +16,8 @@ CONFIG = Config(os.path.dirname(__file__)) DOIT_CONFIG = {{'verbosity':2, 'backend':'sqlite3'}} PACKAGE_PATH = os.path.basename(CONFIG["package_path"]) IMAGE = "%s/%s:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"]) -LOGGER = logging.getLogger(__name__) +IMAGE_JUPYTER = "%s/%s-jupyter:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"]) +DOCKER_JUPYTER = "Dockerfile.Notebook" PARAMS = [ {{ @@ -246,108 +244,28 @@ def task_web_local(): 'params': PARAMS }} +def task_build_jupyter(): + """Build the Docker image for a Jupyter Lab notebook""" + return {{ + 'basename': 'buildJupyter', + 'actions': ['docker build --tag=%s . -f %s' % (IMAGE_JUPYTER, DOCKER_JUPYTER)], + 'task_dep': ['build'], + 'params': PARAMS + }} def task_jupyter(): - """Run a Jupyter notebook in the Docker container""" - # Allow for auto reload to be enabled and import modules from project package - ipython_config = "c.InteractiveShellApp.extensions.append('autoreload')\n" - ipython_config += "c.InteractiveShellApp.exec_lines = " - ipython_config += "['%autoreload 2', 'import sys', 'sys.path.append(\\'../\\')']" - - # Build the command for running jupyter - command = [ - "pip install -r /app/requirements.txt", - "mkdir /etc/ipython", - "echo \"%s\" > /etc/ipython/ipython_config.py" % ipython_config, - "/usr/local/bin/start.sh jupyter notebook --NotebookApp.token=''" + """Run a Jupyter Lab notebook""" + cmd = [ + "docker", + "run", + "-itp", + "8888:8888", + '-w', + '/app', + "--volume", + "\"%s/\":/app" % CONFIG["volume_path"], + IMAGE_JUPYTER ] - command = "; ".join(command) - - def run_command(): - process = subprocess.Popen( - [ - "docker", - "run", - "--rm", - "--name", - "{project_name}_surround_notebook", - "--volume", - "%s:/app" % CONFIG['volume_path'], - "-p", - "55910:8888", - "--user", - "root", - "-w", - "/app", - "jupyter/base-notebook:307ad2bb5fce", - "bash", - "-c", - command - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8') - - LOGGER.info("Starting jupyter notbook server...\n") - - # Get the IP address of the container, otherwise use localhost - ip_process = subprocess.Popen( - ['docker-machine', 'ip'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8') - ip_process.wait() - - ip_output = ip_process.stdout.readline().rstrip() - - if re.match(r"^(\d{{1,3}}\.){{3}}\d{{1,3}}$", ip_output): - host = ip_output - else: - host = "localhost" - - # Wait for the notebook server to be up before loading browser - while True: - line = process.stderr.readline().rstrip() - if line and 'Serving notebooks from local directory' in line: - break - - if process.poll(): - LOGGER.error("Failed to start the server, please try again!") - - # Stop any containers that might be running - process = subprocess.Popen( - [ - 'docker', - 'stop', - '{project_name}_surround_notebook' - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait() - return - - # Open the browser automatically - webbrowser.open('http://%s:55910/tree' % host, new=2) - - LOGGER.info("Notebook URL: http://%s:55910/tree\n", host) - LOGGER.info("Use CTRL+C to stop the server.") - - try: - process.wait() - except KeyboardInterrupt: - pass - finally: - LOGGER.info("Closing server...") - process = subprocess.Popen( - [ - 'docker', - 'stop', - '{project_name}_surround_notebook' - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait() - return {{ - 'actions': [run_command] + 'actions': [" ".join(cmd)], }}