[MatrixManager] Add node expiration management

octobot_evaluators/data/matrix.pxd

@@ -22,7 +22,7 @@ cdef class Matrix:
     cdef public str matrix_id
     cdef public EventTree matrix
 
-    cpdef void set_node_value(self, object value, object value_type, list value_path)
+    cpdef void set_node_value(self, object value, object value_type, list value_path, double timestamp=*)
     cpdef list get_node_children_at_path(self, list node_path, EventTreeNode starting_node=*)
     cpdef dict get_node_children_by_names_at_path(self, list node_path, EventTreeNode starting_node=*)
     cpdef EventTreeNode get_node_at_path(self, list node_path, EventTreeNode starting_node=*)

octobot_evaluators/data/matrix.py

@@ -31,23 +31,48 @@ def __init__(self):
         self.matrix_id = str(uuid.uuid4())
         self.matrix = EventTree()
 
-    def set_node_value(self, value, value_type, value_path):
-        self.matrix.set_node_at_path(value, value_type, value_path)
+    def set_node_value(self, value, value_type, value_path, timestamp=0):
+        """
+        Set the node value at node path
+        :param value_path: the node path
+        :param value_type: the node type
+        :param value: the node value
+        :param timestamp: the value modification timestamp.
+        """
+        self.matrix.set_node_at_path(value, value_type, value_path, timestamp=timestamp)
 
     def get_node_children_at_path(self, node_path, starting_node=None):
+        """
+        Get the node children list
+        :param node_path: the node path
+        :param starting_node: the node to start the relative path
+        :return: the list of node children
+        """
         try:
             return list(self.matrix.get_node(node_path, starting_node=starting_node).children.values())
         except NodeExistsError:
             return []
 
     def get_node_children_by_names_at_path(self, node_path, starting_node=None):
+        """
+        Get the node children dict with node name as key
+        :param node_path: the node path
+        :param starting_node: the node to start the relative path
+        :return: the dict of node children
+        """
         try:
             return {key: val
                     for key, val in self.matrix.get_node(node_path, starting_node=starting_node).children.items()}
         except NodeExistsError:
             return {}
 
     def get_node_at_path(self, node_path, starting_node=None):
+        """
+        Get the EventTreeNode at path
+        :param node_path: the node path
+        :param starting_node: the node to start the relative path
+        :return: the node instance at path
+        """
         try:
             return self.matrix.get_node(node_path, starting_node=starting_node)
         except NodeExistsError:

octobot_evaluators/data_manager/matrix_manager.pxd

@@ -18,7 +18,8 @@ from octobot_commons.event_tree cimport EventTreeNode
 from octobot_evaluators.data.matrix cimport Matrix
 
 cpdef Matrix get_matrix(str matrix_id)
-cpdef void set_tentacle_value(str matrix_id, list tentacle_path, object tentacle_type, object tentacle_value)
+cpdef void set_tentacle_value(str matrix_id, list tentacle_path, object tentacle_type,
+                              object tentacle_value, double timestamp=*)
 cpdef EventTreeNode get_tentacle_node(str matrix_id, list tentacle_path)
 cpdef object get_tentacle_value(str matrix_id, list tentacle_path)
 cpdef list get_matrix_default_value_path(str tentacle_name,
@@ -27,11 +28,9 @@ cpdef list get_matrix_default_value_path(str tentacle_name,
                                   str cryptocurrency=*,
                                   str symbol=*,
                                   str time_frame=*)
-
 cpdef list get_tentacle_nodes(str matrix_id, str exchange_name=*, object tentacle_type=*, str tentacle_name=*)
 cpdef list get_tentacles_value_nodes(str matrix_id, list tentacle_nodes, str cryptocurrency=*, str symbol=*, str time_frame=*)
-
 cpdef list get_tentacle_path(str exchange_name=*, object tentacle_type=*, str tentacle_name=*)
 cpdef list get_tentacle_value_path(str cryptocurrency=*, str symbol=*, str time_frame=*)
-
-
+cpdef bint is_tentacle_value_valid(str matrix_id, list tentacle_path, double timestamp=*, int delta=*)
+cpdef bint is_tentacles_values_valid(str matrix_id, list tentacle_path_list, double timestamp=*, int delta=*)

octobot_evaluators/data_manager/matrix_manager.py

@@ -14,6 +14,11 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
 import asyncio
+import time
+
+from octobot_commons.constants import MINUTE_TO_SECONDS, MIN_EVAL_TIME_FRAME
+
+from octobot_commons.enums import TimeFrames, TimeFramesMinutes
 
 from octobot_evaluators.matrices.matrices import Matrices
 
@@ -27,15 +32,17 @@ def get_matrix(matrix_id):
     return Matrices.instance().get_matrix(matrix_id)
 
 
-def set_tentacle_value(matrix_id, tentacle_path, tentacle_type, tentacle_value):
+def set_tentacle_value(matrix_id, tentacle_path, tentacle_type, tentacle_value, timestamp=0):
     """
     Set the node value at tentacle path
     :param matrix_id: the matrix id
     :param tentacle_path: the tentacle path
     :param tentacle_type: the tentacle type
     :param tentacle_value: the tentacle value
+    :param timestamp: the value modification timestamp.
     """
-    get_matrix(matrix_id).set_node_value(value=tentacle_value, value_type=tentacle_type, value_path=tentacle_path)
+    get_matrix(matrix_id).set_node_value(value=tentacle_value, value_type=tentacle_type,
+                                         value_path=tentacle_path, timestamp=timestamp)
 
 
 def get_tentacle_node(matrix_id, tentacle_path):
@@ -189,3 +196,35 @@ async def subscribe_nodes_event(matrix_id, nodes_path, callback, timeout=None):
     """
     await get_nodes_event(matrix_id, nodes_path, timeout=timeout)
     callback()
+
+
+def is_tentacle_value_valid(matrix_id, tentacle_path, timestamp=0, delta=10) -> bool:
+    """
+    # TODO This method only works with complete default path
+    Check if the node is ready to be used
+    :param matrix_id: the matrix id
+    :param tentacle_path: the tentacle node path
+    :param timestamp: the timestamp to use
+    :param delta: the authorized delta to be valid (in seconds)
+    :return: True if the node is valid else False
+    """
+    if timestamp == 0:
+        timestamp = time.time()
+    return timestamp - (get_tentacle_node(matrix_id, tentacle_path).node_value_time +
+                        TimeFramesMinutes[TimeFrames(tentacle_path[-1])] * MINUTE_TO_SECONDS + delta) < 0
+
+
+def is_tentacles_values_valid(matrix_id, tentacle_path_list, timestamp=0, delta=10) -> bool:
+    """
+    Check if each of the tentacle path value is valid
+    :param matrix_id: the matrix id
+    :param tentacle_path_list: the tentacle node path list
+    :param timestamp: the timestamp to use
+    :param delta: the authorized delta to be valid (in seconds)
+    :return: True if all the node values are valid else False
+    """
+    return all(is_tentacle_value_valid(matrix_id=matrix_id,
+                                       tentacle_path=tentacle_path,
+                                       timestamp=timestamp,
+                                       delta=delta)
+               for tentacle_path in tentacle_path_list)

tests/data_manager/test_matrix_manager.py

@@ -14,13 +14,18 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
 import asyncio
+import time
 
 import pytest
+from octobot_commons.constants import MINUTE_TO_SECONDS
+
+from octobot_commons.enums import TimeFramesMinutes, TimeFrames
 
 from octobot_evaluators.data.matrix import Matrix
 from octobot_evaluators.data_manager.matrix_manager import get_tentacle_path, get_tentacle_value_path, \
     get_tentacle_nodes, get_tentacles_value_nodes, get_matrix_default_value_path, set_tentacle_value, \
-    get_tentacle_value, get_nodes_event, get_nodes_clear_event, get_tentacle_node
+    get_tentacle_value, get_nodes_event, get_nodes_clear_event, get_tentacle_node, \
+    is_tentacle_value_valid, is_tentacles_values_valid
 from octobot_evaluators.matrices.matrices import Matrices
 
 
@@ -485,3 +490,95 @@ async def test_get_nodes_clear_event():
                 for node_path in
                 [evaluator_1_path, evaluator_2_path, evaluator_3_path, evaluator_4_path, evaluator_5_path]])
     Matrices.instance().del_matrix(matrix.matrix_id)
+
+
+@pytest.mark.asyncio
+async def test_is_tentacle_value_valid():
+    matrix = Matrix()
+    Matrices.instance().add_matrix(matrix)
+
+    evaluator_1_path = get_matrix_default_value_path(tentacle_type="TA", tentacle_name="Test-TA",
+                                                     cryptocurrency="BTC",
+                                                     symbol="BTC/USD",
+                                                     time_frame="1m")
+    evaluator_2_path = get_matrix_default_value_path(tentacle_type="TA", tentacle_name="Test-TA",
+                                                     cryptocurrency="BTC",
+                                                     symbol="BTC/USD",
+                                                     time_frame="1h")
+
+    # simulate AbstractEvaluator.initialize()
+    set_tentacle_value(matrix.matrix_id, evaluator_1_path, "TA", None)
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None)
+
+    get_tentacle_node(matrix.matrix_id, evaluator_1_path).node_value_time = time.time()
+    assert is_tentacle_value_valid(matrix.matrix_id, evaluator_1_path)
+    assert not is_tentacle_value_valid(matrix.matrix_id, evaluator_2_path)
+
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None, timestamp=100)
+    assert not is_tentacle_value_valid(matrix.matrix_id, evaluator_2_path)
+
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_HOUR] * 2 * MINUTE_TO_SECONDS)
+    assert not is_tentacle_value_valid(matrix.matrix_id, evaluator_2_path)
+
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_HOUR] * MINUTE_TO_SECONDS)
+    assert is_tentacle_value_valid(matrix.matrix_id, evaluator_2_path)
+
+    # test delta
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_HOUR] * MINUTE_TO_SECONDS - 10)
+    assert not is_tentacle_value_valid(matrix.matrix_id, evaluator_2_path)
+
+    # test delta
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_HOUR] * MINUTE_TO_SECONDS - 9)
+    assert is_tentacle_value_valid(matrix.matrix_id, evaluator_2_path)
+
+    # test modified delta
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_HOUR] * MINUTE_TO_SECONDS - 29)
+    assert is_tentacle_value_valid(matrix.matrix_id, evaluator_2_path, delta=30)
+
+    # test modified delta
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_HOUR] * MINUTE_TO_SECONDS - 31)
+    assert not is_tentacle_value_valid(matrix.matrix_id, evaluator_2_path, delta=30)
+
+    Matrices.instance().del_matrix(matrix.matrix_id)
+
+
+@pytest.mark.asyncio
+async def test_is_tentacles_values_valid():
+    matrix = Matrix()
+    Matrices.instance().add_matrix(matrix)
+
+    evaluator_1_path = get_matrix_default_value_path(tentacle_type="TA", tentacle_name="Test-TA",
+                                                     cryptocurrency="BTC",
+                                                     symbol="BTC/USD",
+                                                     time_frame="1m")
+    evaluator_2_path = get_matrix_default_value_path(tentacle_type="TA", tentacle_name="Test-TA",
+                                                     cryptocurrency="BTC",
+                                                     symbol="BTC/USD",
+                                                     time_frame="1h")
+
+    # simulate AbstractEvaluator.initialize()
+    set_tentacle_value(matrix.matrix_id, evaluator_1_path, "TA", None)
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None)
+
+    assert not is_tentacles_values_valid(matrix.matrix_id, [evaluator_1_path, evaluator_2_path])
+
+    set_tentacle_value(matrix.matrix_id, evaluator_1_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_MINUTE] * 2 * MINUTE_TO_SECONDS)
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_HOUR] * 2 * MINUTE_TO_SECONDS)
+    assert not is_tentacles_values_valid(matrix.matrix_id, [evaluator_1_path, evaluator_2_path])
+
+    set_tentacle_value(matrix.matrix_id, evaluator_1_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_MINUTE] * MINUTE_TO_SECONDS)
+    assert not is_tentacles_values_valid(matrix.matrix_id, [evaluator_1_path, evaluator_2_path])
+
+    set_tentacle_value(matrix.matrix_id, evaluator_2_path, "TA", None,
+                       timestamp=time.time() - TimeFramesMinutes[TimeFrames.ONE_HOUR] * MINUTE_TO_SECONDS)
+    assert is_tentacles_values_valid(matrix.matrix_id, [evaluator_1_path, evaluator_2_path])
+    Matrices.instance().del_matrix(matrix.matrix_id)