Add Jupyter notebook example recipe

docs/building_blocks.md

@@ -468,6 +468,9 @@ __Parameters__
 - __channels__: List of additional Conda channels to enable.  The
 default is an empty list.
 
+- __environment__: Path to the Conda environment file to clone.  The
+default value is empty.
+
 - __eula__: By setting this value to `True`, you agree to the [Anaconda End User License Agreement](https://docs.anaconda.com/anaconda/eula/).
 The default value is `False`.
 
@@ -497,6 +500,10 @@ conda(packages=['numpy'])
 conda(channels=['conda-forge', 'nvidia'], prefix='/opt/conda')
 ```
 
+```python
+conda(environment='environment.yml')
+```
+
 
 ## runtime
 ```python
@@ -2552,6 +2559,9 @@ an empty list.
 
 - __pip__: The name of the `pip` tool to use. The default is `pip`.
 
+- __requirements__: Path to pip requirements file.  The default is
+empty.
+
 - __upgrade__: Boolean flag to control whether pip itself should be
 upgraded prior to installing any PyPi packages.  The default is
 False.
@@ -2567,6 +2577,10 @@ pip(packages=['hpccm'])
 pip(packages=['hpccm'], pip='pip3')
 ```
 
+```python
+pip(requirements='requirements.txt')
+```
+
 
 # pnetcdf
 ```python

hpccm/building_blocks/conda.py

@@ -46,6 +46,9 @@ class conda(bb_base, hpccm.templates.rm, hpccm.templates.wget):
     channels: List of additional Conda channels to enable.  The
     default is an empty list.
 
+    environment: Path to the Conda environment file to clone.  The
+    default value is empty.
+
     eula: By setting this value to `True`, you agree to the [Anaconda End User License Agreement](https://docs.anaconda.com/anaconda/eula/).
     The default value is `False`.
 
@@ -74,6 +77,10 @@ class conda(bb_base, hpccm.templates.rm, hpccm.templates.wget):
     conda(channels=['conda-forge', 'nvidia'], prefix='/opt/conda')
     ```
 
+    ```python
+    conda(environment='environment.yml')
+    ```
+
     """
 
     def __init__(self, **kwargs):
@@ -85,6 +92,7 @@ def __init__(self, **kwargs):
         self.__baseurl = kwargs.get('baseurl',
                                     'http://repo.anaconda.com/miniconda')
         self.__channels = kwargs.get('channels', [])
+        self.__environment = kwargs.get('environment', None)
 
         # By setting this value to True, you agree to the
         # corresponding Anaconda End User License Agreement
@@ -118,6 +126,9 @@ def __instructions(self):
 
         self += comment('Anaconda')
         self += packages(ospackages=self.__ospackages)
+        if self.__environment:
+            self += copy(src=self.__environment, dest=posixpath.join(
+                self.__wd, posixpath.basename(self.__environment)))
         self += shell(commands=self.__commands)
 
     def __cpu_arch(self):
@@ -158,7 +169,7 @@ def __setup(self):
             posixpath.join(self.__prefix, 'etc', 'profile.d', 'conda.sh')))
 
         # Activate
-        if self.__channels or self.__packages:
+        if self.__channels or self.__environment or self.__packages:
             self.__commands.append('. {}'.format(
                 posixpath.join(self.__prefix, 'etc', 'profile.d', 'conda.sh')))
             self.__commands.append('conda activate base')
@@ -169,6 +180,15 @@ def __setup(self):
                 ' '.join(['--add channels {}'.format(x)
                           for x in sorted(self.__channels)])))
 
+        # Install environment
+        if self.__environment:
+            self.__commands.append('conda env update -f {}'.format(
+                posixpath.join(self.__wd,
+                               posixpath.basename(self.__environment))))
+            self.__commands.append(self.cleanup_step(
+                items=[posixpath.join(
+                    self.__wd, posixpath.basename(self.__environment))]))
+
         # Install conda packages
         if self.__packages:
             self.__commands.append('conda install -y {}'.format(

hpccm/building_blocks/pip.py

@@ -23,16 +23,19 @@
 
 from distutils.version import StrictVersion
 import logging # pylint: disable=unused-import
+import posixpath
 
 import hpccm.config
+import hpccm.templates.rm
 
 from hpccm.building_blocks.base import bb_base
 from hpccm.building_blocks.packages import packages
 from hpccm.common import linux_distro
 from hpccm.primitives.comment import comment
+from hpccm.primitives.copy import copy
 from hpccm.primitives.shell import shell
 
-class pip(bb_base):
+class pip(bb_base, hpccm.templates.rm):
     """The `pip` building block installs Python packages from PyPi.
 
     # Parameters
@@ -52,6 +55,9 @@ class pip(bb_base):
 
     pip: The name of the `pip` tool to use. The default is `pip`.
 
+    requirements: Path to pip requirements file.  The default is
+    empty.
+
     upgrade: Boolean flag to control whether pip itself should be
     upgraded prior to installing any PyPi packages.  The default is
     False.
@@ -66,6 +72,10 @@ class pip(bb_base):
     pip(packages=['hpccm'], pip='pip3')
     ```
 
+    ```python
+    pip(requirements='requirements.txt')
+    ```
+
     """
 
     def __init__(self, **kwargs):
@@ -78,7 +88,9 @@ def __init__(self, **kwargs):
         self.__ospackages = kwargs.get('ospackages', None)
         self.__packages = kwargs.get('packages', [])
         self.__pip = kwargs.get('pip', 'pip')
+        self.__requirements = kwargs.get('requirements', None)
         self.__upgrade = kwargs.get('upgrade', False)
+        self.__wd = '/var/tmp' # working directory
 
         self.__debs = [] # Filled in below
         self.__rpms = [] # Filled in below
@@ -122,6 +134,19 @@ def __instructions(self):
             if self.__upgrade:
                 cmds.append('{0} install --upgrade pip'.format(self.__pip))
 
+            if self.__requirements:
+                self += copy(src=self.__requirements,
+                             dest=posixpath.join(
+                                 self.__wd,
+                                 posixpath.basename(self.__requirements)))
+                cmds.append('{0} install -r {1}'.format(
+                    self.__pip,
+                    posixpath.join(self.__wd,
+                                   posixpath.basename(self.__requirements))))
+                cmds.append(self.cleanup_step(items=[
+                    posixpath.join(self.__wd,
+                                   posixpath.basename(self.__requirements))]))
+
             if self.__packages:
                 cmds.append('{0} install {1}'.format(self.__pip,
                                                      ' '.join(self.__packages)))

recipes/jupyter/.gitignore

@@ -0,0 +1,2 @@
+Dockerfile
+Singularity.def

recipes/jupyter/README.md

@@ -0,0 +1,53 @@
+# Containerizing Jupyter Notebooks
+
+This example demonstrates how to generate a container for a Jupyter
+notebook.
+
+The `jupyter.py` script outputs a container specification file that
+can be built into a container image.  Use `--help` to list all the
+script command line options.
+
+The example includes a very simple Jupyter notebook, Conda environment
+(`environment.yml`), and `requirements.txt` for illustrative purposes.
+
+## Docker
+
+Using pip:
+
+```
+$ jupyter.py --notebook notebook.py --requirements requirements.txt > Dockerfile
+```
+
+Using Anaconda:
+
+```
+$ jupyter.py --packager anaconda --notebook notebook.py --environment environment.yml > Dockerfile
+```
+
+Once the Dockerfile has been generated, the steps to build and run
+the container are the same.
+
+```
+$ sudo docker build -t jupyter:example -f Dockerfile .
+$ sudo docker run --rm -p 8888:8888 jupyter:example
+```
+
+## Singularity
+
+Using pip:
+
+```
+$ jupyter.py --notebook notebook.py --requirements requirements.txt --format singularity > Singularity.def
+```
+
+Using Anaconda:
+$ jupyter.py --packager anaconda --notebook notebook.py --environment.yml --format singularity > Singularity.def
+```
+
+Once the Singularity definition file has been generated the steps to
+build and run the container are the same.
+
+```
+$ sudo singularity build jupyter-example.sif Singularity.def
+$ singularity run jupyter-example.sif
+```

recipes/jupyter/environment.yml

@@ -0,0 +1,4 @@
+channels:
+  - conda-forge
+dependencies:
+  - conda-forge::hpccm

recipes/jupyter/jupyter.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+
+import argparse
+import hpccm
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Jupyter notebook container generator')
+    parser.add_argument('--environment', type=str,
+                        help='Conda environment file')
+    parser.add_argument('--format', type=str, default='docker',
+                        choices=['docker', 'singularity'],
+                        help='Container specification format (default: docker)')
+    parser.add_argument('--image', type=str, default='ubuntu:18.04',
+                        help='Base container image (default: ubuntu:18.04)')
+    parser.add_argument('--notebook', required=True, type=str,
+                        help='Jupyter notebook file')
+    parser.add_argument('--packager', type=str, default='pip',
+                        choices=['anaconda', 'pip'],
+                        help='Python package manager (default: pip)')
+    parser.add_argument('--requirements', type=str,
+                        help='pip requirements file')
+    args = parser.parse_args()
+
+    ### Create Stage
+    stage = hpccm.Stage()
+
+    ### Base image
+    stage += hpccm.primitives.baseimage(image=args.image, _docker_env=False)
+
+    ### Install Python and Jupyter (and requirements / environment)
+    if args.packager == 'pip':
+        stage += hpccm.building_blocks.python(python2=False)
+        stage += hpccm.building_blocks.pip(packages=['ipython', 'jupyter'],
+                                           pip='pip3',
+                                           requirements=args.requirements)
+    elif args.packager == 'anaconda':
+        stage += hpccm.building_blocks.conda(environment=args.environment,
+                                             eula=True,
+                                             packages=['ipython', 'jupyter'])
+
+    ### Make the port accessible (Docker only)
+    stage += hpccm.primitives.raw(docker='EXPOSE 8888')
+
+    ### Add the notebook itself
+    stage += hpccm.primitives.copy(src=args.notebook, dest='/notebook/',
+                                   _mkdir=True)
+
+    ### Run Jupyter
+    if args.packager == 'pip':
+        stage += hpccm.primitives.runscript(
+            commands=['jupyter notebook --no-browser --ip 0.0.0.0 --notebook-dir /notebook --allow-root'])
+    elif args.packager == 'anaconda':
+        stage += hpccm.primitives.shell(commands=[
+            'echo "#!/bin/bash\\nsource /usr/local/anaconda/bin/activate base\\njupyter notebook --ip 0.0.0.0 --no-browser --notebook-dir /notebook --allow-root" > /usr/local/bin/entrypoint.sh',
+            'chmod a+x /usr/local/bin/entrypoint.sh'])
+        stage += hpccm.primitives.runscript(
+            commands=['/usr/local/bin/entrypoint.sh'])
+
+    ### Set container specification output format
+    hpccm.config.set_container_format(args.format)
+
+    ### Output container specification
+    print(stage)
+

recipes/jupyter/notebook.ipynb

@@ -0,0 +1,43 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Example Jupyter notebook container\n",
+    "\n",
+    "Hello World!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!hpccm --version"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

recipes/jupyter/requirements.txt

@@ -0,0 +1 @@
+hpccm

test/test_conda.py

@@ -119,6 +119,31 @@ def test_channels(self):
     /usr/local/anaconda/bin/conda clean -afy && \
     rm -rf /var/tmp/Miniconda3-4.7.12-Linux-x86_64.sh''')
 
+    @x86_64
+    @ubuntu
+    @docker
+    def test_environment(self):
+        """environment"""
+        c = conda(eula=True, environment='foo/environment.yml')
+        self.assertEqual(str(c),
+r'''# Anaconda
+RUN apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        ca-certificates \
+        wget && \
+    rm -rf /var/lib/apt/lists/*
+COPY foo/environment.yml /var/tmp/environment.yml
+RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp http://repo.anaconda.com/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh && \
+    bash /var/tmp/Miniconda3-4.7.12-Linux-x86_64.sh -b -p /usr/local/anaconda && \
+    /usr/local/anaconda/bin/conda init && \
+    ln -s /usr/local/anaconda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    . /usr/local/anaconda/etc/profile.d/conda.sh && \
+    conda activate base && \
+    conda env update -f /var/tmp/environment.yml && \
+    rm -rf /var/tmp/environment.yml && \
+    /usr/local/anaconda/bin/conda clean -afy && \
+    rm -rf /var/tmp/Miniconda3-4.7.12-Linux-x86_64.sh''')
+
     @x86_64
     @ubuntu
     @docker