Merge pull request #43 from Anaconda-Platform/fusion-details

Add registers_fusion_function:true if a notebook contains @fusion.register

anaconda_project/commands/test/test_command_commands.py

@@ -224,7 +224,7 @@ def check_guessing_notebook(dirname):
 
     with_directory_contents_completing_project_file(
         {DEFAULT_PROJECT_FILENAME: 'packages:\n - notebook\n',
-         'file.ipynb': ""}, check_guessing_notebook)
+         'file.ipynb': "{}"}, check_guessing_notebook)
 
 
 def test_add_command_with_env_spec(monkeypatch, capsys):

anaconda_project/internal/notebook_analyzer.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# ----------------------------------------------------------------------------
+# Copyright © 2017, Continuum Analytics, Inc. All rights reserved.
+#
+# The full license is in the file LICENSE.txt, distributed with this software.
+# ----------------------------------------------------------------------------
+"""Analyze notebook files."""
+from __future__ import absolute_import
+
+import codecs
+import json
+import re
+
+from anaconda_project.internal.py2_compat import is_string
+
+_comment_re = re.compile("#.*$", re.MULTILINE)
+_fusion_register_re = re.compile(r"^\s*@fusion\.register", re.MULTILINE)
+
+
+# see if some source has @fusion.register. This is
+# obviously sort of heuristic, but without executing
+# the python we can only do so much.
+def _has_fusion_register(source):
+    # dump comments so commenting out fusion.register
+    # would work as expected
+    source = re.sub(_comment_re, "", source)
+    return re.match(_fusion_register_re, source) is not None
+
+
+def extras(filename, errors):
+    try:
+        with codecs.open(filename, encoding='utf-8') as f:
+            json_string = f.read()
+            parsed = json.loads(json_string)
+    except Exception as e:
+        errors.append("Failed to read or parse %s: %s" % (filename, str(e)))
+        return None
+
+    extras = dict()
+    found_fusion = False
+
+    if isinstance(parsed, dict) and \
+       'cells' in parsed and \
+       isinstance(parsed['cells'], list):
+        for cell in parsed['cells']:
+            if 'source' in cell:
+                if isinstance(cell['source'], list):
+                    source = "".join([s for s in cell['source'] if is_string(s)])
+                    if _has_fusion_register(source):
+                        found_fusion = True
+
+    if found_fusion:
+        extras['registers_fusion_function'] = True
+
+    return extras

anaconda_project/internal/test/test_notebook_analyzer.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# ----------------------------------------------------------------------------
+# Copyright © 2017, Continuum Analytics, Inc. All rights reserved.
+#
+# The full license is in the file LICENSE.txt, distributed with this software.
+# ----------------------------------------------------------------------------
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import os
+
+import anaconda_project.internal.notebook_analyzer as notebook_analyzer
+
+from anaconda_project.internal.test.tmpfile_utils import with_directory_contents
+
+
+def _fake_notebook_json_with_code(source):
+    ipynb = {
+        "cells": [
+            {"cell_type": "code",
+             # source is a list of lines with the newline included in each
+             "source": [(s + "\n") for s in source.split("\n")]}
+        ]
+    }
+    return ipynb
+
+
+def _with_code_in_notebook_file(source, f):
+    def check(dirname):
+        filename = os.path.join(dirname, "foo.ipynb")
+        return f(filename)
+
+    json_string = json.dumps(_fake_notebook_json_with_code(source))
+    with_directory_contents({"foo.ipynb": json_string}, check)
+
+
+def test_extras_with_simple_has_fusion_register():
+    def check(filename):
+        errors = []
+        extras = notebook_analyzer.extras(filename, errors)
+        assert [] == errors
+        assert extras == {'registers_fusion_function': True}
+
+    _with_code_in_notebook_file("""
+@fusion.register
+def some_func():
+   pass
+    """, check)
+
+
+def test_extras_without_has_fusion_register():
+    def check(filename):
+        errors = []
+        extras = notebook_analyzer.extras(filename, errors)
+        assert [] == errors
+        assert extras == {}
+
+    _with_code_in_notebook_file("""
+def some_func():
+   pass
+    """, check)
+
+
+def test_fusion_register():
+    assert not notebook_analyzer._has_fusion_register("")
+    assert not notebook_analyzer._has_fusion_register("fusion.register\n")
+    assert notebook_analyzer._has_fusion_register("@fusion.register\n")
+    assert notebook_analyzer._has_fusion_register("    @fusion.register\n")
+    assert not notebook_analyzer._has_fusion_register("# @fusion.register\n")
+    assert notebook_analyzer._has_fusion_register("# foo\n@fusion.register\n")
+    assert notebook_analyzer._has_fusion_register("# foo\n\n    @fusion.register\n")
+    assert notebook_analyzer._has_fusion_register("@fusion.register # foo\n")
+    assert notebook_analyzer._has_fusion_register("""
+@fusion.register(args=blah)
+def some_func():
+   pass
+""")
+    assert not notebook_analyzer._has_fusion_register("""
+# @fusion.register
+def some_func():
+   pass
+""")
+
+
+def test_extras_with_io_error(monkeypatch):
+    def mock_codecs_open(*args, **kwargs):
+        raise IOError("Nope")
+
+    monkeypatch.setattr('codecs.open', mock_codecs_open)
+    errors = []
+    extras = notebook_analyzer.extras("blah", errors)
+    assert [] != errors
+    assert extras is None
+    assert 'Failed to read or parse' in errors[0]

anaconda_project/project.py

@@ -17,14 +17,15 @@
 from anaconda_project.plugins.requirements.conda_env import CondaEnvRequirement
 from anaconda_project.plugins.requirements.download import DownloadRequirement
 from anaconda_project.plugins.requirements.service import ServiceRequirement
-from anaconda_project.project_commands import ProjectCommand
+from anaconda_project.project_commands import (ProjectCommand, all_known_command_attributes)
 from anaconda_project.project_file import ProjectFile
 from anaconda_project.archiver import _list_relative_paths_for_unignored_project_files
 from anaconda_project.version import version
 
 from anaconda_project.internal.py2_compat import is_string
 from anaconda_project.internal.simple_status import SimpleStatus
 from anaconda_project.internal.slugify import slugify
+import anaconda_project.internal.notebook_analyzer as notebook_analyzer
 import anaconda_project.internal.conda_api as conda_api
 import anaconda_project.internal.pip_api as pip_api
 
@@ -591,9 +592,7 @@ def _update_commands(self, problems, project_file, requirements):
                     failed = True
                     continue
 
-                _unknown_field_suggestions(project_file, problems, attrs,
-                                           ('description', 'env_spec', 'supports_http_options', 'bokeh_app', 'notebook',
-                                            'unix', 'windows', 'conda_app_entry'))
+                _unknown_field_suggestions(project_file, problems, attrs, all_known_command_attributes)
 
                 if 'description' in attrs and not is_string(attrs['description']):
                     _file_problem(problems, project_file,
@@ -617,6 +616,11 @@ def _update_commands(self, problems, project_file, requirements):
                                       (attrs['env_spec'], name))
                         failed = True
 
+                if 'registers_fusion_function' in attrs and not isinstance(attrs['registers_fusion_function'], bool):
+                    _file_problem(problems, project_file,
+                                  ("'registers_fusion_function' field of command {} must be a boolean".format(name)))
+                    failed = True
+
                 copied_attrs = deepcopy(attrs)
 
                 if 'env_spec' not in copied_attrs:
@@ -716,7 +720,15 @@ def need_to_import_notebook(relative_name):
 
         def make_add_notebook_func(relative_name, env_spec_name):
             def add_notebook(project):
+                errors = []
+                extras = notebook_analyzer.extras(os.path.join(self.directory_path, relative_name), errors)
+                # TODO this is broken, need to refactor so fix functions can return
+                # errors and probably also log progress indication.
+                assert [] == errors
+                assert extras is not None
+
                 command_dict = {'notebook': relative_name, 'env_spec': env_spec_name}
+                command_dict.update(extras)
                 project.project_file.set_value(['commands', relative_name], command_dict)
 
             return add_notebook
@@ -1133,6 +1145,7 @@ def publication_info(self):
                 commands[key]['default'] = True
             commands[key]['env_spec'] = command.default_env_spec_name
             commands[key]['supports_http_options'] = command.supports_http_options
+            commands[key].update(command.extras)
         json['commands'] = commands
         envs = dict()
         for key, env in self.env_specs.items():

anaconda_project/project_commands.py

@@ -28,6 +28,11 @@
 except ImportError:  # pragma: no cover (py2 only)
     from urllib import quote as url_quote  # pragma: no cover (py2 only)
 
+standard_command_attributes = ('description', 'env_spec', 'supports_http_options', 'bokeh_app', 'notebook', 'unix',
+                               'windows', 'conda_app_entry')
+extra_command_attributes = ('registers_fusion_function', )
+all_known_command_attributes = standard_command_attributes + extra_command_attributes
+
 
 def _is_windows():
     # it's tempting to cache this but it hoses our test monkeypatching so don't.
@@ -407,6 +412,19 @@ def description(self):
         assert description is not None
         return description
 
+    @property
+    def extras(self):
+        """Dictionary of extra attributes not covered by other properties.
+
+        These are typically 'plugin specific' (only for notebook, only for bokeh,
+        etc.)
+        """
+        result = dict()
+        for k in self._attributes.keys():
+            if k in extra_command_attributes:
+                result[k] = self._attributes[k]
+        return result
+
     def _choose_args_and_shell(self, environ, extra_args=None):
         assert extra_args is None or isinstance(extra_args, list)
 

anaconda_project/project_ops.py

@@ -26,6 +26,7 @@
 from anaconda_project.internal.simple_status import SimpleStatus
 import anaconda_project.conda_manager as conda_manager
 from anaconda_project.internal.conda_api import parse_spec
+import anaconda_project.internal.notebook_analyzer as notebook_analyzer
 
 _default_projectignore = """
 # project-local contains your personal configuration choices and state
@@ -848,6 +849,19 @@ def add_command(project, name, command_type, command, env_spec_name=None, suppor
         assert isinstance(supports_http_options, bool)
         command_dict['supports_http_options'] = supports_http_options
 
+    if command_type == 'notebook':
+        notebook_file = os.path.join(project.directory_path, command)
+        errors = []
+        # TODO missing notebook should be an error caught before here
+        if os.path.isfile(notebook_file):
+            extras = notebook_analyzer.extras(notebook_file, errors)
+        else:
+            extras = {}
+        if len(errors) > 0:
+            failed = SimpleStatus(success=False, description="Unable to add the command.", errors=errors)
+            return failed
+        command_dict.update(extras)
+
     project.project_file.use_changes_without_saving()
 
     failed = project.problems_status(description="Unable to add the command.")