IGNITE-13829: added rotate log

modules/ducktests/tests/ignitetest/services/ignite.py

@@ -83,7 +83,7 @@ def get_event_time(service, log_node, log_pattern, from_the_beginning=True, time
                                 backoff_sec=0.3)
 
     _, stdout, _ = log_node.account.ssh_client.exec_command(
-        "grep '%s' %s" % (log_pattern, IgniteAwareService.STDOUT_STDERR_CAPTURE))
+        "grep '%s' %s" % (log_pattern, log_node.log_file))
 
     return datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read().decode("utf-8")).group(),
                              "[%Y-%m-%d %H:%M:%S,%f]")

modules/ducktests/tests/ignitetest/services/ignite_app.py

@@ -102,7 +102,7 @@ def extract_results(self, name):
 
         for node in self.nodes:
             output = node.account.ssh_capture(
-                "grep '%s' %s" % (name + "->", self.STDOUT_STDERR_CAPTURE), allow_fail=False)
+                "grep '%s' %s" % (name + "->", node.log_file), allow_fail=False)
             for line in output:
                 res.append(re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1))
 

modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py

@@ -83,6 +83,8 @@ def await_started(self):
     def start_node(self, node):
         self.init_persistent(node)
 
+        self.__update_node_log_file(node)
+
         super().start_node(node)
 
         wait_until(lambda: self.alive(node), timeout_sec=10)
@@ -188,7 +190,7 @@ def pids(self, node):
 
     # pylint: disable=W0613
     def _worker(self, idx, node):
-        cmd = self.spec.command
+        cmd = self.spec.command(node.log_file)
 
         self.logger.debug("Attempting to start Application Service on %s with command: %s" % (str(node.account), cmd))
 
@@ -201,7 +203,8 @@ def alive(self, node):
         """
         return len(self.pids(node)) > 0
 
-    def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5):
+    @staticmethod
+    def await_event_on_node(evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5):
         """
         Await for specific event message in a node's log file.
         :param evt_message: Event message.
@@ -211,7 +214,7 @@ def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning
         :param backoff_sec: Number of seconds to back off between each failure to meet the condition
                 before checking again.
         """
-        with monitor_log(node, self.STDOUT_STDERR_CAPTURE, from_the_beginning) as monitor:
+        with monitor_log(node, node.log_file, from_the_beginning) as monitor:
             monitor.wait_until(evt_message, timeout_sec=timeout_sec, backoff_sec=backoff_sec,
                                err_msg="Event [%s] was not triggered on '%s' in %d seconds" % (evt_message, node.name,
                                                                                                timeout_sec))
@@ -351,3 +354,13 @@ def __dump_netfilter_settings(node):
         Reads current netfilter settings on the node for debugging purposes.
         """
         return str(node.account.ssh_client.exec_command("sudo iptables -L -n")[1].read(), sys.getdefaultencoding())
+
+    def __update_node_log_file(self, node):
+        """
+        Update the node log file.
+        """
+        cnt = list(node.account.ssh_capture(f'ls {self.LOGS_DIR} | '
+                                            f'grep -E "^console_[0-9]*.log$" | '
+                                            f'wc -l', callback=int))[0]
+
+        node.log_file = self.STDOUT_STDERR_CAPTURE.replace('.log', f'_{cnt + 1}.log')

modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py

@@ -28,12 +28,13 @@ class PersistenceAware:
     """
     # Root directory for persistent output
     PERSISTENT_ROOT = "/mnt/service"
-    STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log")
     TEMP_DIR = os.path.join(PERSISTENT_ROOT, "tmp")
+    LOGS_DIR = os.path.join(PERSISTENT_ROOT, "logs")
+    STDOUT_STDERR_CAPTURE = os.path.join(LOGS_DIR, "console.log")
 
     logs = {
         "console_log": {
-            "path": STDOUT_STDERR_CAPTURE,
+            "path": LOGS_DIR,
             "collect_default": True
         }
     }
@@ -45,6 +46,7 @@ def init_persistent(self, node):
         """
         node.account.mkdirs(self.PERSISTENT_ROOT)
         node.account.mkdirs(self.TEMP_DIR)
+        node.account.mkdirs(self.LOGS_DIR)
 
 
 class IgnitePersistenceAware(PersistenceAware):

modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py

@@ -74,7 +74,6 @@ def config_template(self):
             return IgniteClientConfigTemplate()
         return IgniteServerConfigTemplate()
 
-    @property
     @abstractmethod
     def command(self):
         """
@@ -100,14 +99,14 @@ class IgniteNodeSpec(IgniteSpec, IgnitePersistenceAware):
     """
     Spec to run ignite node
     """
-    @property
-    def command(self):
+    # pylint: disable=W0221
+    def command(self, stdout_stderr):
         cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \
               (self._envs(),
                self.path.script("ignite.sh"),
                self._jvm_opts(),
                self.CONFIG_FILE,
-               self.STDOUT_STDERR_CAPTURE)
+               stdout_stderr)
 
         return cmd
 
@@ -123,14 +122,14 @@ def __init__(self, **kwargs):
     def _app_args(self):
         return ",".join(self.args)
 
-    @property
-    def command(self):
+    # pylint: disable=W0221
+    def command(self, stdout_stderr):
         cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \
               (self._envs(),
                self.path.script("ignite.sh"),
                self._jvm_opts(),
                self._app_args(),
-               self.STDOUT_STDERR_CAPTURE)
+               stdout_stderr)
 
         return cmd
 

modules/ducktests/tests/ignitetest/tests/discovery_test.py

@@ -244,13 +244,13 @@ def _simulate_and_detect_failure(self, servers, failed_nodes, timeout):
 
     def _check_failed_number(self, failed_nodes, survived_node):
         """Ensures number of failed nodes is correct."""
-        cmd = "grep '%s' %s | wc -l" % (node_failed_event_pattern(), IgniteAwareService.STDOUT_STDERR_CAPTURE)
+        cmd = "grep '%s' %s | wc -l" % (node_failed_event_pattern(), survived_node.log_file)
 
         failed_cnt = int(str(survived_node.account.ssh_client.exec_command(cmd)[1].read(), sys.getdefaultencoding()))
 
         if failed_cnt != len(failed_nodes):
             failed = str(survived_node.account.ssh_client.exec_command(
-                "grep '%s' %s" % (node_failed_event_pattern(), IgniteAwareService.STDOUT_STDERR_CAPTURE))[1].read(),
+                "grep '%s' %s" % (node_failed_event_pattern(), survived_node.log_file))[1].read(),
                          sys.getdefaultencoding())
 
             self.logger.warn("Node '%s' (%s) has detected the following failures:%s%s" % (
@@ -263,7 +263,7 @@ def _check_not_segmented(self, failed_nodes):
         """Ensures only target nodes failed"""
         for service in [srv for srv in self.test_context.services if isinstance(srv, IgniteAwareService)]:
             for node in [srv_node for srv_node in service.nodes if srv_node not in failed_nodes]:
-                cmd = "grep -i '%s' %s | wc -l" % ("local node segmented", IgniteAwareService.STDOUT_STDERR_CAPTURE)
+                cmd = "grep -i '%s' %s | wc -l" % ("local node segmented", node.log_file)
 
                 failed = str(node.account.ssh_client.exec_command(cmd)[1].read(), sys.getdefaultencoding())