Adds module meta for transforming Lena sequences. Adds SetContext and…

… UpdateContextFromStatic meta elements. LenaSequence supports static context, which allows to set output directories and Cache names during the initialisation phase.

Technical changes: LenaSequence gets _full_seq field.
lena.context does not depend on lena.flow (get_data_context) to avoid the circular dependency.

docs/source/meta.rst

@@ -0,0 +1,15 @@
+Meta
+======
+**Elements:**
+
+.. currentmodule:: lena.meta
+.. autosummary::
+
+    SetContext
+    UpdateContextFromStatic
+
+Elements
+--------
+
+.. autoclass:: SetContext
+.. autoclass:: UpdateContextFromStatic

docs/source/reference.rst

@@ -9,6 +9,7 @@ Reference
     flow <flow>
     input <input>
     math <math>
+    meta <meta>
     output <output>
     structures <structures>
     variables <variables>

lena/context/context.py

@@ -3,7 +3,7 @@
 import json
 
 import lena.core
-import lena.flow
+# import lena.flow
 
 
 class Context(dict):
@@ -83,7 +83,8 @@ def __call__(self, value):
         its initialization argument *d*
         has no effect on the produced values.
         """
-        data, context = lena.flow.get_data_context(value)
+        data, context = value
+        # data, context = lena.flow.get_data_context(value)
         return (data, Context(context))
 
     def __getattr__(self, name):

lena/context/update_context.py

@@ -4,7 +4,7 @@
 
 import lena.core
 import lena.context
-import lena.flow
+# import lena.flow
 
 
 _sentinel = object()
@@ -173,7 +173,8 @@ def __call__(self, value):
         the update argument is missing in *value*'s context.
         """
         import jinja2
-        data, context = lena.flow.get_data_context(value)
+        data, context = value
+        # data, context = lena.flow.get_data_context(value)
         if isinstance(self._update, (str, jinja2.Template)):
             if self._context_value:
                 if not self._has_default:

lena/core/fill_request_seq.py

@@ -73,6 +73,8 @@ def __init__(self, *args, **kwargs):
         self._reset = fr._reset
         # fr = adapters.FillRequest(self, reset=reset, bufsize=self._bufsize)
         self.run = fr.run
+        # todo: we will abandon FillRequestSeq in the next release
+        super(FillRequestSeq, self).__init__(*self._seq)
 
     def fill(self, value):
         """Fill *self* with *value*.

lena/core/lena_sequence.py

@@ -1,4 +1,7 @@
 """LenaSequence abstract base class."""
+from copy import deepcopy
+
+import lena.context
 
 
 class LenaSequence(object):
@@ -9,16 +12,51 @@ class LenaSequence(object):
     get its length and get an item at the given index.
     """
     def __init__(self, *args):
-        self._seq = args
+        # todo: this doesn't require jinja2,
+        # but we may want it to support template strings soon
+        from lena.context import update_recursively
+        self._full_seq = args
+        seq = []
+
+        # for static (sequence initialisation time) context
+        need_context = []
+        context = {}
+
+        # get static context.
+        # External/earlier context takes precedence.
+        for el in reversed(args):
+            # we don't check whether they are callable here,
+            # because otherwise there will be an error
+            # during initialisation (not runtime)
+            if hasattr(el, "_get_context"):
+                # todo: allow template substitution
+                update_recursively(context, el._get_context())
+
+        for el in args:
+            # orders of setters and getters are independent:
+            # context is same for the whole sequence
+            # (and external ones, but not for Split)
+            if hasattr(el, "_set_context"):
+                el._set_context(context)
+                need_context.append(el)
+
+            # todo: or has context
+            if not hasattr(el, "_has_no_data"):
+                seq.append(el)
+
+        self._context = context
+        self._need_context = need_context
+
+        self._seq = seq
 
     def __iter__(self):
-        return self._seq.__iter__()
+        return self._full_seq.__iter__()
 
     def __len__(self):
-        return self._seq.__len__()
+        return self._full_seq.__len__()
 
     def __getitem__(self, ind):
-        return self._seq[ind]
+        return self._full_seq[ind]
 
     def _repr_nested(self, base_indent="", indent=" "*4, el_separ=",\n"):
         # to get a one-line representation, use el_separ=", ", indent=""
@@ -31,9 +69,9 @@ def repr_maybe_nested(el, base_indent, indent):
 
         elems = el_separ.join((repr_maybe_nested(el, base_indent=base_indent,
                                                  indent=indent)
-                               for el in self._seq))
+                               for el in self._full_seq))
 
-        if "\n" in el_separ and self._seq:
+        if "\n" in el_separ and self._full_seq:
             # maybe new line
             mnl = "\n"
             # maybe base indent
@@ -48,3 +86,12 @@ def __repr__(self):
         # maybe: make a compact representation with repr
         # and nested with str
         return self._repr_nested()
+
+    def _set_context(self, context):
+        for el in self._need_context:
+            # we don't use self._context,
+            # because it was already set in this seq.
+            el._set_context(context)
+
+    def _get_context(self):
+        return deepcopy(self._context)

lena/core/sequence.py

@@ -26,9 +26,15 @@ def __init__(self, *args):
         For more information about the *run* method and callables,
         see :class:`Run`.
         """
+        # _name is used for representation.
+        # Subclass must set its own name or provide a different repr
+        self._name = "Sequence"
+        super(Sequence, self).__init__(*args)
+
         seq = []
 
-        for el in args:
+        # todo: we could change self._seq in place
+        for el in self._seq:
             if hasattr(el, "run") and callable(el.run):
                 seq.append(el)
             else:
@@ -44,17 +50,14 @@ def __init__(self, *args):
                     )
                 else:
                     seq.append(run_el)
-        # _name is used for representation.
-        # Subclass must set its own name or provide a different repr
-        self._name = "Sequence"
 
-        super(Sequence, self).__init__(*seq)
+        self._seq = seq
 
     def run(self, flow):
-        """Generator, which transforms the incoming flow.
+        """Generator that transforms the incoming flow.
 
         If this :class:`Sequence` is empty,
-        the flow passes untransformed,
+        the flow passes unaltered, but
         with a small change.
         This function converts input flow to an iterator,
         so that it always contains both *iter* and *next* methods.
@@ -63,7 +66,7 @@ def run(self, flow):
         """
         flow = functions.flow_to_iter(flow)
 
-        for el in self:
+        for el in self._seq:
             flow = el.run(flow)
 
         flow = functions.flow_to_iter(flow)

lena/meta/__init__.py

@@ -0,0 +1 @@
+from .elements import SetContext, UpdateContextFromStatic

lena/meta/elements.py

@@ -0,0 +1,63 @@
+from copy import deepcopy
+
+import lena.context
+from lena.context import update_recursively
+
+
+class SetContext(object):
+    """Set static context for this sequence.
+
+    Static context does not automatically update runtime context.
+    Use :class:`UpdateContextFromStatic` for that.
+
+    Static context can be used during the initialisation phase
+    to set output directories, :class:`.Cache` names, etc.
+    There is no way to update static context from runtime one.
+    """
+
+    def __init__(self, key, value):
+        """*key* is a string representing a
+        (possibly nested) dictionary key. *value* is its value.
+        See :func:`.str_to_dict` for details.
+        """
+
+        # todo: key could be a complete dictionary
+        # self._subcontext = lena.context.str_to_list(key)
+        # self._value = value
+        self._key = key
+        self._value = value
+        self._context = lena.context.str_to_dict(key, value)
+        self._has_no_data = True
+
+    def _get_context(self):
+        return deepcopy(self._context)
+
+    def __repr__(self):
+        val = self._value
+        if isinstance(val, str):
+            val = '"' + val + '"'
+        return 'SetContext("{}", {})'.format(self._key, val)
+
+
+class UpdateContextFromStatic(object):
+    """Update runtime context with the static one.
+
+    Note that for runtime context later elements
+    update previous values, but for static context
+    it is the opposite (external and previous elements
+    take precedence).
+    """
+
+    def __init__(self):
+        self._context = {}
+
+    def _set_context(self, context):
+        self._context = context
+
+    def run(self, flow):
+        for val in flow:
+            data, context = val
+            # no template substitutions are done,
+            # that would be too complicated, fragile and wrong
+            update_recursively(context, deepcopy(self._context))
+            yield (data, context)

tests/context/test_update_context.py

@@ -181,7 +181,7 @@ def test_update_context_exception_strings():
     # test exception strings
     uc = UpdateContext("var", "{{data}}", raise_on_missing=True)
     with pytest.raises(lena.core.LenaKeyError, match="'data' is undefined, context={}"):
-        uc({})
+        uc((None, {}))
     with pytest.raises(
         lena.core.LenaValueError,
         match="fix braces for template string '{{{data}}' or set value to False"

tests/meta/test_elements.py

@@ -0,0 +1,42 @@
+from copy import deepcopy
+
+import pytest
+
+from lena.core import Sequence, Source
+
+from lena.meta.elements import SetContext, UpdateContextFromStatic
+
+
+def test_set_context():
+    data = [(0, {})]
+    set_context_far = SetContext("data.detector", "far")
+
+    # repr works
+    assert repr(set_context_far) == 'SetContext("data.detector", "far")'
+
+    ## static context does not alter context
+    # within a sequence
+    s0 = Sequence(set_context_far)
+    assert list(s0.run(deepcopy(data))) == data
+
+    # outside a sequence
+    set_context_near = SetContext("data.detector", "near")
+    s1 = Sequence(set_context_near, s0)
+    assert list(s1.run(deepcopy(data))) == data
+
+    # static context is updated after the element
+    s0u0 = Sequence(set_context_far, UpdateContextFromStatic())
+    res0u0 = list(s0u0.run(deepcopy(data)))
+    assert len(res0u0) == len(data)
+    assert res0u0[0][0] == data[0][0]
+    assert res0u0[0][1] == {'data': {'detector': 'far'}}
+
+    # static context is updated after the sequence
+    s0u1 = Sequence(s0, UpdateContextFromStatic())
+    res0u1 = list(s0u1.run(deepcopy(data)))
+    assert res0u1[0][1] == {'data': {'detector': 'far'}}
+
+    # external context overwrites internal (further) one
+    s1u0 = Sequence(s1, UpdateContextFromStatic())
+    res1u0 = list(s1u0.run(deepcopy(data)))
+    assert res1u0[0][1] == {'data': {'detector': 'near'}}