pypyr.steps.pyimport. includes all plumbing to make it work per conte…

…xt instance. pypyr.steps.contextclearall also wipes pyimports. closes #110.

pypyr/cache/namespacecache.py

@@ -0,0 +1,47 @@
+"""Global cache for namespace imports.
+
+Attributes:
+    pystring_namespace_cache: global instance of the namespace cache for
+        PyString expressions. Use this attribute to access the cache from
+        elsewhere.
+"""
+import logging
+from sys import intern
+from pypyr.cache.cache import Cache
+from pypyr.moduleloader import ImportVisitor
+
+
+logger = logging.getLogger(__name__)
+
+
+class NamespaceCache(Cache):
+    """Cache of namespace dictionaries.
+
+    Parse source string for python import statements using AST Visitor.
+    """
+
+    def get_namespace(self, source):
+        """Get cached namespace. Adds to cache if not exist.
+
+        Args:
+            source (str): String with python 'import x.y'/'from x import y'
+                statements.
+
+        Returns:
+            Namespace dictionary of imported references.
+        """
+        logger.debug("starting")
+
+        # source can be relatively long.
+        # interning means cache dict compare obj id rather than full str parse
+        interned_source = intern(source)
+        namespace = self.get(
+            interned_source,
+            lambda: ImportVisitor().get_namespace(interned_source))
+
+        logger.debug("done")
+        return namespace
+
+
+# global instance of the cache. use this to access the cache from elsewhere.
+pystring_namespace_cache = NamespaceCache()

pypyr/context.py

@@ -27,6 +27,8 @@ class Context(dict):
         pipeline_name (str): name of pipeline that is currently running
         working_dir (path-like): working directory path. Either CWD or
                                  initialized from the cli --dir arg.
+        pystring_globals (dict): globals namespace for PyString expression
+                                 evals.
 
     """
 
@@ -36,6 +38,11 @@ class Context(dict):
     # https://github.com/python/cpython/blob/master/Lib/string.py
     formatter = RecursiveFormatter(special_types=SpecialTagDirective)
 
+    def __init__(self, *args, **kwargs):
+        """Initialize context."""
+        super().__init__(*args, **kwargs)
+        self.pystring_globals = {}
+
     def __missing__(self, key):
         """Throw KeyNotInContextError rather than KeyError.
 
@@ -200,7 +207,7 @@ def assert_keys_type_value(self,
             self.assert_key_type_value(context_item, caller, extra_error_text)
 
     def get_eval_string(self, input_string):
-        """Dynamically evaluates the input_string expression.
+        """Dynamically evaluates the input_string python expression.
 
         This provides dynamic python eval of an input expression. The return is
         whatever the result of the expression is.
@@ -221,7 +228,15 @@ def get_eval_string(self, input_string):
             Whatever object results from the string expression valuation.
 
         """
-        return expressions.eval_string(input_string, self)
+        if input_string:
+            return expressions.eval_string(input_string,
+                                           self.pystring_globals,
+                                           self)
+        else:
+            # Empty input raises cryptic EOF syntax err, this more human
+            # friendly
+            raise ValueError('input expression is empty. It must be a valid '
+                             'python expression instead.')
 
     def get_formatted(self, key):
         """Return formatted value for context[key].

pypyr/dsl.py

@@ -16,7 +16,7 @@
 # use pypyr logger to ensure loglevel is set correctly
 logger = logging.getLogger(__name__)
 
-# ------------------------ custom yaml tags -----------------------------------
+# region custom yaml tags
 
 
 class SpecialTagDirective:
@@ -199,7 +199,9 @@ class PyString(SpecialTagDirective):
     def get_value(self, context):
         """Run python eval on the input string."""
         if self.value:
-            return expressions.eval_string(self.value, context)
+            return expressions.eval_string(self.value,
+                                           context.pystring_globals,
+                                           context)
         else:
             # Empty input raises cryptic EOF syntax err, this more human
             # friendly
@@ -230,7 +232,7 @@ def get_value(self, context=None):
         """Simply return the string as is, the whole point of a sic string."""
         return self.value
 
-# ------------------------ END custom yaml tags -------------------------------
+# endregion custom yaml tags
 
 
 class Step:

pypyr/moduleloader.py

@@ -1,11 +1,11 @@
-"""pypyr dynamic modules and path discovery.
+"""pypyr dynamic modules, namespaces and path discovery.
 
 Load modules dynamically, find things on file-system.
 
 Attributes:
     working_dir (WorkingDir): Global shared current working dir.
 """
-
+import ast
 import importlib
 import logging
 from pathlib import Path
@@ -16,6 +16,105 @@
 logger = logging.getLogger(__name__)
 
 
+class ImportVisitor(ast.NodeVisitor):
+    """Parse python import and import from syntax.
+
+    Use this to parse python syntax of import/import from, import the modules
+    and save the result to the imported_namespace namespace dictionary.
+
+    Only supports absolute imports, not relative. Does not support wildcard
+    style 'from x import *'. But you weren't planning on doing *that* anyway,
+    I hope.
+
+    Supports source like:
+        import x
+        import x as y
+        import x.y
+        import x.y as z
+        from x import y
+        from x import y as z
+        from x import y, z
+        from a.b import c as d, e as f
+
+    Usage example:
+        ImportVisitor().visit(ast.parse('from mod.sub import attr as alias'))
+
+    Will result in:
+        imported_namespace == {'alias': <<attr in mod.sub module>>}
+
+    Attributes:
+        imported_namespace (dict): Namespace dictionary of imported references.
+    """
+
+    def __init__(self):
+        """Initialize me."""
+        self.imported_namespace = {}
+
+    def get_namespace(self, source):
+        """Parse source, import modules & return namespace.
+
+        You might as well call this instead of visit().
+
+        Args:
+            source (str): String of Python source code.
+
+        Return:
+            Namespace dictionary of imported references.
+        """
+        self.visit(ast.parse(source))
+        return self.imported_namespace
+
+    def _set_namespace(self, alias, obj):
+        """Add imported object to namespace dictionary.
+
+        Args:
+            alias (ast.alias): Use asname for alias if it exists, else fall
+                back to name.
+            obj (any): Imported module or attribute like class or function.
+
+        Returns: None
+        """
+        as_name = alias.asname if alias.asname else alias.name
+        self.imported_namespace[as_name] = obj
+
+    def visit_Import(self, node):
+        """Process syntax nodes of form: import module."""
+        for alias in node.names:
+            if alias.asname:
+                imported_module = importlib.import_module(alias.name)
+            else:
+                # if no alias, 'import mod.sub' has to bind 'mod' to the
+                # imported parent obj.
+                parent, dot, _ = alias.name.partition('.')
+                if dot:
+                    # mod.sub1.sub2 should save to namespace as parent 'mod'
+                    alias.asname = parent
+                    # __import__(), although discouraged, is how to get the
+                    # top-level module - this is the obj that is bound by the
+                    # name in the namespace
+                    imported_module = __import__(alias.name)
+                else:
+                    imported_module = importlib.import_module(alias.name)
+
+            self._set_namespace(alias, imported_module)
+
+    def visit_ImportFrom(self, node):
+        """Process syntax nodes of form: from parent import child."""
+        if node.level > 0:
+            raise TypeError("you can't use relative imports here. "
+                            "use absolute imports instead.")
+        imported_module = importlib.import_module(node.module)
+        for alias in node.names:
+            try:
+                imported_obj = getattr(imported_module, alias.name)
+            except AttributeError:
+                # if no attribute, might be form: from mod import submod
+                imported_obj = importlib.import_module(
+                    f'{node.module}.{alias.name}')
+
+            self._set_namespace(alias, imported_obj)
+
+
 class WorkingDir():
     """The Working Directory.
 

pypyr/steps/contextclearall.py

@@ -6,7 +6,7 @@
 
 
 def run_step(context):
-    """Wipe the entire context.
+    """Wipe the entire context, including the pyimport namespace.
 
     Args:
         Context is a dictionary or dictionary-like.
@@ -15,6 +15,8 @@ def run_step(context):
     logger.debug("started")
 
     context.clear()
-    logger.info("Context wiped. New context size: %s", len(context))
+    context.pystring_globals.clear()
+    logger.info("context & py imports wiped. New context size: %s",
+                len(context))
 
     logger.debug("done")

pypyr/steps/pyimport.py

@@ -0,0 +1,54 @@
+"""pypyr step that imports to the namespace for PyString.
+
+Make Python stdlib or your own code available to any PyString in the pipeline.
+
+Only supports absolute imports, not relative.
+
+Supports python import syntax like:
+    import x
+    import x as y
+
+    import x.y
+    import x.y as z
+
+    from x import y
+    from x import y as z
+
+    from x import y, z
+    from a.b import c as d, e as f
+"""
+import logging
+from pypyr.cache.namespacecache import pystring_namespace_cache
+
+logger = logging.getLogger(__name__)
+
+
+def run_step(context):
+    """Import modules, classes or functions for PyString expressions.
+
+    Args:
+        context (pypyr.context.Context):
+            Context is a dictionary or dictionary-like.
+            Context must contain key 'pyimport'
+            The value of pyimport is a string containing python import
+            statements.
+    """
+    logger.debug("started")
+    context.assert_key_has_value(key='pyImport', caller=__name__)
+
+    # block yaml style could be LiteralScalarString, which is not hashable for
+    # cache key. Explicitly make a string.
+    source = str(context['pyImport'])
+
+    namespace = pystring_namespace_cache.get_namespace(source)
+    # len safe coz get_namespace returns {}, not None.
+    logger.debug("imported %s objects to merge into PyString namespace",
+                 len(namespace))
+
+    context.pystring_globals.update(namespace)
+
+    # pystring_globals initialized to {} on Context init, so len() is safe.
+    logger.debug("PyString namespace now contains %s objects.",
+                 len(context.pystring_globals))
+
+    logger.debug("done")

pypyr/utils/expressions.py

@@ -1,7 +1,7 @@
 """Utility functions for evaluating expressions."""
 
 
-def eval_string(input_string, locals):
+def eval_string(input_string, globals, locals):
     """Dynamically evaluates the input_string expression.
 
     This provides dynamic python eval of an input expression. The return is
@@ -30,4 +30,4 @@ def eval_string(input_string, locals):
 
     """
     # empty globals arg will append __builtins__ by default
-    return eval(input_string, {}, locals)
+    return eval(input_string, globals, locals)

tests/arbpack/arbmod2.py

@@ -0,0 +1,6 @@
+"""Arbitrary module with a test attribute."""
+
+
+def arb_func_in_arbmod2(arg):
+    """Arbitrary function just returns input."""
+    return arg

tests/arbpack/arbmod3.py

@@ -0,0 +1,6 @@
+"""Arbitrary module with a test attribute."""
+
+
+def arb_func_in_arbmod3(arg):
+    """Arbitrary function just returns input."""
+    return arg

tests/arbpack/arbmod4_avoid.py

@@ -0,0 +1,15 @@
+"""Arbitrary module with a test attribute.
+
+DON'T use me anywhere except moduleloader_test.py > test_import_visitor.
+
+Because otherwise coverage will drop <100% since Python import machinery will
+cache the module lookup and test_import_visitor explicitly tests a code-path
+where a module is not found in cache.
+
+Believe it or not, this is the least painful way of achieving it.
+"""
+
+
+def arb_func_in_arbmod4(arg):
+    """Arbitrary function just returns input."""
+    return arg

tests/arbpack/arbmultiattr.py

@@ -0,0 +1,8 @@
+"""Arbitrary module with multiple attributes."""
+
+arb_attr = 123.456
+
+
+def arb_func(arg):
+    """Arbitrary function just returns input."""
+    return arg

tests/integration/pypyr/pype/pype_int_test.py

@@ -19,3 +19,12 @@ def test_pype_err_int():
     test_pipe_runner.assert_pipeline_notify_output_is(pipename, ['A',
                                                                  'B',
                                                                  'C'])
+
+
+def test_pype_pyimport():
+    """Pype while passing pyimports to child."""
+    pipename = 'pype/pyimport/parent'
+    test_pipe_runner.assert_pipeline_notify_output_is(pipename, ['A',
+                                                                 'B',
+                                                                 'C',
+                                                                 'D'])