More export options (#19)

Export fixes

Dockerfile

@@ -26,4 +26,6 @@ EXPOSE 8888
 WORKDIR /code
 VOLUME /code
 COPY tests/handler.ipynb example.ipynb
+COPY tests/env.txt .
+ENV ENV_FILE=/code/env.txt
 CMD jupyter lab --core-mode --allow-root --no-browser --ip 0.0.0.0

docs/magic.rst

@@ -15,6 +15,7 @@ Overview
    In [1]: %nuclio help Overview
    Show help on command. Available commands:
     - cmd: Run a command, add it to "build
+    - config: Set function configuration value
     - env: Set environment variable
     - env_file: Set environment from file(s)
     - export: Export notebook
@@ -42,6 +43,23 @@ cmd
 
     In [3]: %nuclio cmd --config-only apt-get install -y libyaml-dev
 
+config
+------
+
+::
+
+   In [1]: %nuclio help config
+   Set function configuration value. Values need to be Python literals (1,
+    "debug", 3.3 ...). You can use += to append values to a list
+
+    Example:
+    In [1] %nuclio config spec.maxReplicas = 5
+    In [2]: %%nuclio config
+    ...: spec.maxReplicas = 5
+    ...: spec.runtime = "python2.7"
+    ...: build.commands +=  "apk --update --no-cache add ca-certificates"
+    ...:
+
 env
 ---
 
@@ -83,7 +101,16 @@ export
 ::
 
    In [1]: %nuclio help export
-   Export notebook.
+   Export notebook. Possible options are:
+
+    --output-dir path
+        Output directory path
+    --notebook path
+        Path to notebook file
+    --handler-name name
+        Name of handler
+    --handler-file path
+        Path to handler code (Python file)
 
     Example:
     In [1] %nuclio export
@@ -92,14 +119,18 @@ export
     Notebook exported to handler at '/tmp/handler'
     In [3] %nuclio export --notebook /path/to/notebook.ipynb
     Notebook exported to handler at '/tmp/nuclio-handler-29803'
+    In [4] %nuclio export --handler-name faces
+    Notebook exported to handler at '/tmp/nuclio-handler-29804'
+    In [5] %nuclio export --handler-file /tmp/faces.py
+    Notebook exported to handler at '/tmp/nuclio-handler-29805'
 
 handler
 -------
 
 ::
 
    In [1]: %nuclio help handler
-   Mark this cell as handler function.
+   Mark this cell as handler function. You can give optional name
 
     %%nuclio handler
     ctx.logger.info('handler called')

nuclio/export.py

@@ -17,17 +17,17 @@
 from collections import namedtuple
 from datetime import datetime
 from io import BytesIO, StringIO
-from os.path import abspath, dirname
+from os import path, environ
 from textwrap import indent
 from zipfile import ZipFile
 
 import yaml
 from nbconvert.exporters import Exporter
 from nbconvert.filters import ipython2python
 
-from .utils import iter_env_lines, parse_config_line, parse_env
+from .utils import env_keys, iter_env_lines, parse_config_line, parse_env
 
-here = dirname(abspath(__file__))
+here = path.dirname(path.abspath(__file__))
 
 Magic = namedtuple('Magic', 'name is_cell args lines')
 magic_handlers = {}  # name -> function
@@ -36,7 +36,7 @@
 is_return = re.compile(r'#\s*nuclio:\s*return').search
 # # nuclio: ignore
 has_ignore = re.compile(r'#\s*nuclio:\s*ignore').search
-handler_decl = 'def handler(context, event):'
+handler_decl = 'def {}(context, event):'
 indent_prefix = '    '
 missing = object()
 
@@ -46,14 +46,16 @@
     'metadata': {},
     'spec': {
         'runtime': 'python:3.6',
-        'handler': 'handler:handler',
+        'handler': None,
         'env': [],
     },
     'build': {
         'commands': [],
     }
 }
 
+handlers = []
+
 
 class NuclioExporter(Exporter):
     """Export to nuclio handler"""
@@ -68,6 +70,8 @@ def _file_extension_default(self):
         return '.zip'
 
     def from_notebook_node(self, nb, resources=None, **kw):
+        function_config['spec']['handler'] = handler_name()
+
         io = StringIO()
         print(self.header(), file=io)
 
@@ -214,11 +218,17 @@ def add_return(line):
 
 @magic_handler
 def handler(magic):
-    return handler_code('\n'.join(magic.lines))
+    name = magic.args[0] if magic.args else next_handler_name()
+    if env_keys.handler_name not in environ:
+        module, _ = function_config['spec']['handler'].split(':')
+        function_config['spec']['handler'] = '{}:{}'.format(module, name)
 
+    code = '\n'.join(magic.lines)
+    return handler_code(name, code)
 
-def handler_code(code):
-    lines = [handler_decl]
+
+def handler_code(name, code):
+    lines = [handler_decl.format(name)]
     code = indent(code, indent_prefix)
     for line in code.splitlines():
         if is_return(line):
@@ -237,13 +247,9 @@ def handler_code(code):
     return '\n'.join(lines)
 
 
-def comment_code(code):
-    return re.sub('^', '# ', code, flags=re.MULTILINE)
-
-
 @magic_handler
 def export(magic):
-    return comment_code('\n'.join(magic.lines))
+    return ''
 
 
 @magic_handler
@@ -274,3 +280,33 @@ def update_in(obj, key, value, append=False):
         obj[last_key].append(value)
     else:
         obj[last_key] = value
+
+
+def next_handler_name():
+    if handlers:
+        name = 'handler_{}'.format(len(handlers))
+    else:
+        name = 'handler'
+    handlers.append(name)
+    return name
+
+
+def module_name(py_file):
+    """
+    >>> module_name('/path/to/handler.py')
+    'handler'
+    """
+    base = path.basename(py_file)
+    module, _ = path.splitext(base)
+    return module
+
+
+def handler_name():
+    handler_path = environ.get(env_keys.handler_path)
+    if handler_path:
+        module = module_name(handler_path)
+    else:
+        module = 'handler'
+
+    name = environ.get(env_keys.handler_name, 'handler')
+    return '{}:{}'.format(module, name)

nuclio/magic.py

@@ -14,26 +14,27 @@
 
 import json
 import re
-import shlex
-from argparse import ArgumentParser
 from glob import glob
 from os import environ, path
-from shutil import unpack_archive
+from shutil import copy, unpack_archive
 from subprocess import run
 from sys import executable, stderr
 from tempfile import mkdtemp
 from urllib.parse import urlencode, urljoin
 from urllib.request import urlopen
+from zipfile import ZipFile
 
 import ipykernel
 from IPython import get_ipython
 from IPython.core.magic import register_line_cell_magic
 from notebook.notebookapp import list_running_servers
 
-from .utils import parse_env, iter_env_lines, parse_config_line
+from .utils import (env_keys, iter_env_lines, parse_config_line, parse_env,
+                    parse_export_line)
 
 log_prefix = '%nuclio: '
 
+
 # Make sure we're working when not running under IPython/Jupyter
 kernel = get_ipython()
 if kernel is None:
@@ -245,7 +246,7 @@ def deploy(line, cell):
 
 @command
 def handler(line, cell):
-    """Mark this cell as handler function.
+    """Mark this cell as handler function. You can give optional name
 
     %%nuclio handler
     ctx.logger.info('handler called')
@@ -266,6 +267,10 @@ def handler(context, event):
 # https://github.com/jupyter/notebook/issues/1000#issuecomment-359875246
 def notebook_file_name():
     """Return the full path of the jupyter notebook."""
+    # Check that we're running under notebook
+    if not (kernel and kernel.config['IPKernelApp']):
+        return
+
     kernel_id = re.search('kernel-(.*).json',
                           ipykernel.connect.get_connection_file()).group(1)
     servers = list_running_servers()
@@ -280,7 +285,16 @@ def notebook_file_name():
 
 @command
 def export(line, cell, return_dir=False):
-    """Export notebook.
+    """Export notebook. Possible options are:
+
+    --output-dir path
+        Output directory path
+    --notebook path
+        Path to notebook file
+    --handler-name name
+        Name of handler
+    --handler-file path
+        Path to handler code (Python file)
 
     Example:
     In [1] %nuclio export
@@ -289,11 +303,15 @@ def export(line, cell, return_dir=False):
     Notebook exported to handler at '/tmp/handler'
     In [3] %nuclio export --notebook /path/to/notebook.ipynb
     Notebook exported to handler at '/tmp/nuclio-handler-29803'
+    In [4] %nuclio export --handler-name faces
+    Notebook exported to handler at '/tmp/nuclio-handler-29804'
+    In [5] %nuclio export --handler-file /tmp/faces.py
+    Notebook exported to handler at '/tmp/nuclio-handler-29805'
     """
 
     args, rest = parse_export_line(line)
     if rest:
-        log_error('nunknown arguments: {}'.format(' '.join(rest)))
+        log_error('unknown arguments: {}'.format(' '.join(rest)))
         return
 
     notebook = args.notebook or notebook_file_name()
@@ -303,13 +321,24 @@ def export(line, cell, return_dir=False):
 
     out_dir = args.output_dir or mkdtemp(prefix='nuclio-handler-')
 
+    env = environ.copy()  # Pass argument to exporter via environment
+    if args.handler_name:
+        env[env_keys.handler_name] = args.handler_name
+
+    if args.handler_path:
+        if not path.isfile(args.handler_path):
+            log_error(
+                'cannot find handler file: {}'.format(args.handler_path))
+            return
+        env[env_keys.handler_path] = args.handler_path
+
     cmd = [
         executable, '-m', 'nbconvert',
         '--to', 'nuclio.export.NuclioExporter',
         '--output-dir', out_dir,
         notebook,
     ]
-    out = run(cmd, capture_output=True)
+    out = run(cmd, env=env, capture_output=True)
     if out.returncode != 0:
         print(out.stdout.decode('utf-8'))
         print(out.stderr.decode('utf-8'), file=stderr)
@@ -321,18 +350,21 @@ def export(line, cell, return_dir=False):
         log_error('cannot find zip files in {}'.format(out_dir))
         return
 
-    unpack_archive(out_files[0], out_dir)
+    zip_file = out_files[0]
+    unpack_archive(zip_file, out_dir)
     log('handler exported to {}'.format(out_dir))
-    if return_dir:
-        return out_dir
 
+    if args.handler_path:
+        copy(args.handler_path, out_dir)
 
-def parse_export_line(line):
-    parser = ArgumentParser(prog='%nuclio')
-    parser.add_argument('--output-dir')
-    parser.add_argument('--notebook')
+        with open(args.handler_path) as fp:
+            code = fp.read()
+        name = path.basename(args.handler_path)
+        with ZipFile(zip_file, 'a') as zf:
+            zf.writestr(name, code)
 
-    return parser.parse_known_args(shlex.split(line))
+    if return_dir:
+        return out_dir
 
 
 def print_first_of(pattern):

nuclio/utils.py

@@ -14,7 +14,14 @@
 
 
 import re
+from argparse import ArgumentParser
 from ast import literal_eval
+import shlex
+
+
+class env_keys:
+    handler_name = 'NUCLIO_HANDLER_NAME'
+    handler_path = 'NUCLIO_HANDLER_PATH'
 
 
 def parse_env(line):
@@ -49,3 +56,16 @@ def parse_config_line(line):
         raise ValueError(line)
 
     return key, op, value
+
+
+def parse_export_line(args):
+    parser = ArgumentParser(prog='%nuclio')
+    parser.add_argument('--output-dir')
+    parser.add_argument('--notebook')
+    parser.add_argument('--handler-name')
+    parser.add_argument('--handler-path')
+
+    if isinstance(args, str):
+        args = shlex.split(args)
+
+    return parser.parse_known_args(args)

tests/conftest.py

@@ -1,5 +1,14 @@
 from os import environ
 from os.path import abspath, dirname
 
+import pytest
+
+import nuclio
+
 here = dirname(abspath(__file__))
 environ['ENV_FILE'] = '{}/env.txt'.format(here)
+
+
+@pytest.fixture
+def clean_handlers():
+    nuclio.export.handlers.clear()