Add template rendering to static context in sequences and Split. Not …

…all details are clear, but tests work.

lena/core/lena_sequence.py

@@ -4,6 +4,33 @@
 import lena.context
 
 
+def _update_unknown_contexts(unknown_contexts, context):
+    """Update values in *unknown_contexts* from *context*
+    and update *context* with the new values.
+    """
+    from lena.context import (
+        format_context, str_to_dict, update_recursively
+    )
+    new_unknowns = []
+    for uc in unknown_contexts:
+        key, value = uc
+        fc = format_context(value)
+        try:
+            rendered = fc(context)
+        except lena.core.LenaKeyError:
+            new_unknowns.append(uc)
+        else:
+            rendered_context = str_to_dict(key, rendered)
+            update_recursively(context, rendered_context)
+
+    # if we could render something, try to render other elements
+    # with the updated context
+    if len(new_unknowns) != len(unknown_contexts):
+        new_unknowns = _update_unknown_contexts(new_unknowns, context)
+
+    return new_unknowns
+
+
 class LenaSequence(object):
     """Abstract base class for all Lena sequences.
 
@@ -20,6 +47,7 @@ def __init__(self, *args):
 
         # for static (sequence initialisation time) context
         need_context = []
+        unknown_contexts = []
         context = {}
 
         # get static context.
@@ -28,23 +56,40 @@ def __init__(self, *args):
             # we don't check whether they are callable here,
             # because otherwise there will be an error
             # during initialisation (not runtime)
+            if hasattr(el, "_unknown_contexts"):
+                # we don't expand in place even if we could,
+                # because external/top context has priority
+                unknown_contexts.extend(el._unknown_contexts)
             if hasattr(el, "_get_context"):
-                # todo: allow template substitution
-                update_recursively(context, el._get_context())
+                el_context = el._get_context()
+                update_recursively(context, el_context)
+
+        # Render that context that we can. Context is updated.
+        unknown_contexts = _update_unknown_contexts(unknown_contexts, context)
+        # There is no way to check (during init)
+        # that all static context was set,
+        # because sequences are allowed to separate/wrap any elements.
 
         for el in args:
             # orders of setters and getters are independent:
             # context is same for the whole sequence
             # (and external ones, but not for Split)
             if hasattr(el, "_set_context"):
-                el._set_context(context)
+                if not unknown_contexts:
+                    # we set context after all unknowns are known.
+                    # el can't have unknown contexts
+                    # (except subsequences of Split);
+                    # at least not contexts
+                    # that shall update the current context
+                    el._set_context(context)
                 need_context.append(el)
 
-            # todo: or has context
+            # todo 0.7: or has context
             if not hasattr(el, "_has_no_data"):
                 seq.append(el)
 
         self._context = context
+        self._unknown_contexts = unknown_contexts
         self._need_context = need_context
 
         self._seq = seq
@@ -90,13 +135,29 @@ def __repr__(self):
     def _set_context(self, context):
         from lena.context import update_recursively
         # parent context doesn't necessarily has our context
-        # (for example, when we are inside Split)
+        # (for example, when we are inside Split).
+        # We update current context with context.
         update_recursively(self._context, context)
-        cont = self._context
+        if hasattr(self, "_unknown_contexts"):
+            # Can be empty, this is fine.
+            # We don't update context with those rendered here,
+            # because we can be in Split. All common unknown contexts
+            # have already updated the common context.
+            # We update the current context, however.
+            _update_unknown_contexts(self._unknown_contexts[:],
+                                     self._context)
+            # containing Sequence may redefine some values,
+            # therefore we need to set unknown_contexts each time;
+            # however, we no longer update the external context.
+            # if unknown_contexts:
+            #     self._unknown_contexts = unknown_contexts
+            #     # don't set context for other elements
+            #     # until all contexts are known
+            #     return
+            # else:
+            #     del self._unknown_contexts
         for el in self._need_context:
-            # we don't use self._context,
-            # because it was already set in this seq.
-            el._set_context(cont)
+            el._set_context(self._context)
 
     def _get_context(self):
         return deepcopy(self._context)

lena/core/split.py

@@ -1,6 +1,8 @@
 """Split data flow and run analysis in parallel."""
+import collections
 import copy
 import itertools
+from copy import deepcopy
 
 import lena.context
 from . import fill_compute_seq
@@ -67,6 +69,48 @@ def _get_seq_with_type(seq, bufsize=None):
     return (seq, seq_type)
 
 
+def _get_uc_intersection(unknown_contexts_seq):
+    from copy import copy as copy_
+    # deque to preserve the order, and because it may be faster
+    # to delete elements from start (if all sequences are different)
+    intersection = collections.deque(unknown_contexts_seq[0])
+
+    def remove_from_deque(ucs, context):
+        # index does not help in deque, it's O(n) (or maybe not at 0?)
+        # https://stackoverflow.com/questions/58152201/time-complexity-deleting-element-of-deque
+        while True:
+            try:
+                ucs.remove(context)
+            except ValueError:
+                break
+
+    for unknown_contexts in unknown_contexts_seq[1:]:
+        # we can't iterate a mutated deque
+        for context in copy_(intersection):
+            remove = False
+            if context not in unknown_contexts:
+                remove = True
+                # todo: if a local (known) context sets the same key,
+                # the result will be different (no intersection here!)
+                # Also need to check cardinality
+                # and other unknown contexts setting the same key.
+                # we could have several copies of this context
+            # elif ...
+            if remove:
+                remove_from_deque(intersection, context)
+
+    return list(intersection)
+
+
+def _remove_uc_intersection(unknown_contexts_seq, intersection):
+    for cont in intersection:
+        # cont can be several times (N) in the intersection,
+        # in that case it will be present in unknown_contexts
+        # N times as well
+        for unknown_contexts in unknown_contexts_seq:
+            unknown_contexts.remove(cont)
+
+
 class Split(object):
     """Split data flow and run analysis in parallel."""
 
@@ -107,6 +151,7 @@ def __init__(self, seqs, bufsize=1000, copy_buf=True):
                 "seqs must be a list of sequences, "
                 "{} provided".format(seqs)
             )
+
         seqs = [meta.alter_sequence(seq) for seq in seqs]
         self._seqs = []
         self._seq_types = []
@@ -149,6 +194,32 @@ def __init__(self, seqs, bufsize=1000, copy_buf=True):
                 )
         self._bufsize = bufsize
 
+        contexts = []
+        unknown_contexts_seq = []
+        # the order of sequences is not important
+        for seq in self._seqs:
+            # first we get all known contexts, then unknown ones
+            if hasattr(seq, "_get_context"):
+                contexts.append(seq._get_context())
+            if hasattr(seq, "_unknown_contexts"):
+                # it is important that we have links to actual lists
+                unknown_contexts_seq.append(seq._unknown_contexts)
+
+        if unknown_contexts_seq and len(unknown_contexts_seq) == len(self._seqs):
+            # otherwise ignore them (the intersection is empty):
+            # they will be set from external context.
+            intersection = _get_uc_intersection(unknown_contexts_seq)
+            self._unknown_contexts = intersection
+            # never update template contexts twice.
+            _remove_uc_intersection(unknown_contexts_seq, intersection)
+
+        # todo: if a template context updates an existing context,
+        # this will be wrong. But what if it is a feature?
+        # Don't do that if you are not sure what you are doing (I'm not)
+        # Maybe we shall remove some keys from the intersection
+        # if this ever becomes a problem.
+        self._context = lena.context.intersection(*contexts)
+
     def __call__(self):
         """Each initialization sequence generates flow.
         After its flow is empty, next sequence is called, etc.
@@ -355,17 +426,14 @@ def repr_maybe_nested(el, base_indent, indent):
                         "([", mnl, elems, mnl, mbi, "])"])
 
     def _get_context(self):
-        # we don't set context during initialisation,
-        # because we may need it at most once
-        # (during its parent sequence initialisation)
-        contexts = []
-        # the order of sequences is not important
-        for seq in self._seqs:
-            if hasattr(seq, "_get_context"):
-                contexts.append(seq._get_context())
-        return lena.context.intersection(*contexts)
+        return deepcopy(self._context)
 
     def _set_context(self, context):
+        # we don't update current context here,
+        # because Split is always within a sequence.
+        # todo
+        # If it is not, it has no external context,
+        # or one must first copy its context before setting a new one.
         for seq in self._seqs:
             if hasattr(seq, "_set_context"):
                 seq._set_context(context)

lena/math/elements.py

@@ -11,7 +11,9 @@
 
 import lena.context
 import lena.core
-from lena.core import LenaTypeError, LenaRuntimeError, LenaZeroDivisionError
+from lena.core import (
+    LenaTypeError, LenaRuntimeError, LenaZeroDivisionError, LenaValueError
+)
 import lena.flow
 
 

lena/meta/elements.py

@@ -26,7 +26,13 @@ def __init__(self, key, value):
         # self._value = value
         self._key = key
         self._value = value
-        self._context = lena.context.str_to_dict(key, value)
+        context = lena.context.str_to_dict(key, value)
+        if isinstance(value, str) and '{{' in value:
+            # need to know other context to render this one
+            self._unknown_contexts = [(key, value)]
+            self._context = {}
+        else:
+            self._context = context
         self._has_no_data = True
 
     def _get_context(self):

tests/meta/test_elements.py

@@ -68,10 +68,13 @@ def test_set_context_split():
             store2,
         ),
         (
+            # common context updates the sequence containing Split
             set_context_common,
             call,
             store3,
             set_context_near,
+            # external context overwrites internal one
+            SetContext("data.cycle", 2),
         ),
     ])
     assert split._get_context() == {'data': {'lost': True}}
@@ -92,3 +95,65 @@ def test_set_context_split():
     }
     # static context is the same for the same level of nesting
     assert store4.context == store1.context
+
+
+def test_set_template_context_split():
+    set_context_far = SetContext("data.detector", "far")
+    set_context_near = SetContext("data.detector", "near")
+    set_context_common = SetContext("data.lost", True)
+    call = lambda _: "we won't run this"
+    store1, store2, store3, store4 = [StoreContext(str(i)) for i in range(1, 5)]
+
+    seq = Sequence(
+        # common context updates the sequence containing Split
+        set_context_common,
+        call,
+        store3,
+        set_context_near,
+        # external context overwrites internal one
+        SetContext("data.cycle", 2),
+        SetContext("cycle", "{{data.cycle}}"),
+        # todo: this is too complicated for now
+        # # this will be overwritten
+        # SetContext("cycle", "{{cycle}}_indeed"),
+        SetContext("detector", "{{data.detector}}"),
+        SetContext("detector", "maybe_{{detector}}"),
+    )
+    # print(seq[-1]._context)
+    assert seq._get_context()["detector"] == "maybe_near"
+
+    split = Split([
+        (
+            set_context_common,
+            call,
+            # data.cycle is set externally (and correctly)
+            SetContext("cycle", "{{data.cycle}}"),
+            # SetContext("cycle", "{{cycle}}_indeed"),
+            set_context_far,
+            store2,
+        ),
+        seq,
+    ])
+    assert split._get_context() == {'data': {'lost': True}}
+
+    s0 = Source(
+        SetContext("data.cycle", 1),
+        call,
+        store1,
+        split,
+        store4
+    )
+    assert store1.context == {'data': {'cycle': 1, 'lost': True}}
+    assert store2.context == {
+        'data': {'cycle': 1, 'detector': 'far', 'lost': True},
+        'cycle': '1',
+    }
+    assert store3.context == {
+        'data': {'cycle': 1, 'detector': 'near', 'lost': True},
+        'detector': 'maybe_near',
+        # todo: maybe this is indended?..
+        'cycle': '2',
+        # 'cycle': '2_indeed',
+    }
+    # static context is the same for the same level of nesting
+    assert store4.context == store1.context