Ignite ducktape control sh

.gitignore

@@ -87,3 +87,4 @@ packages
 *.pyc
 /tests/venv
 modules/ducktests/tests/docker/build/**
+modules/ducktests/tests/.tox

.travis.yml

@@ -50,3 +50,11 @@ matrix:
       dotnet: 3.1.101
       script:
         - dotnet build modules/platforms/dotnet/Apache.Ignite.DotNetCore.sln
+
+    - language: python
+      python:
+        - "3.8"
+      install: pip install tox-travis
+      before_script: cd modules/ducktests/tests
+      script:
+        - tox

bin/control.sh

@@ -1,12 +1,9 @@
 #!/usr/bin/env bash
-if [ ! -z "${IGNITE_SCRIPT_STRICT_MODE:-}" ]
-then
-    set -o nounset
-    set -o errexit
-    set -o pipefail
-    set -o errtrace
-    set -o functrace
-fi
+set -o nounset
+set -o errexit
+set -o pipefail
+set -o errtrace
+set -o functrace
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more

bin/include/build-classpath.sh

@@ -47,14 +47,13 @@ includeToClassPath() {
 
     for file in $1/*
     do
-        if [[ -z "${EXCLUDE_MODULES}" ]] || [[ ${EXCLUDE_MODULES} != *"`basename $file`"* ]]; then
-            echo "$file included"
+        if [[ -z "${EXCLUDE_MODULES:-}" ]] || [[ ${EXCLUDE_MODULES:-} != *"`basename $file`"* ]]; then
             if [ -d ${file} ] && [ -d "${file}/target" ]; then
                 if [ -d "${file}/target/classes" ]; then
                     IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/classes
                 fi
 
-                if [[ -z "${EXCLUDE_TEST_CLASSES}" ]]; then
+                if [[ -z "${EXCLUDE_TEST_CLASSES:-}" ]]; then
                   if [ -d "${file}/target/test-classes" ]; then
                       IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/test-classes
                   fi
@@ -68,7 +67,7 @@ includeToClassPath() {
           echo "$file excluded by EXCLUDE_MODULES settings"
         fi
     done
-    
+
     IFS=$SAVEIFS
 }
 

modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2

@@ -30,7 +30,9 @@
         </property>
 
         <property name="clientMode" value="{{ client_mode or False | lower }}"/>
-
+        {% if consistent_id %}
+            <property name="consistentId" value="{{ consistent_id }}"/>
+        {% endif %}
         {{ properties }}
     </bean>
 </beans>

modules/ducktests/tests/ignitetest/services/utils/control_utility.py

@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains control utility wrapper.
+"""
+import random
+import re
+from collections import namedtuple
+
+from ducktape.cluster.remoteaccount import RemoteCommandError
+
+
+class ControlUtility:
+    """
+    Control utility (control.sh) wrapper.
+    """
+    BASE_COMMAND = "control.sh"
+
+    def __init__(self, cluster, text_context):
+        self._cluster = cluster
+        self.logger = text_context.logger
+
+    def baseline(self):
+        """
+        :return Baseline nodes.
+        """
+        return self.cluster_state().baseline
+
+    def cluster_state(self):
+        """
+        :return: Cluster state.
+        """
+        output = self.__run("--baseline")
+
+        return self.__parse_cluster_state(output)
+
+    def set_baseline(self, baseline):
+        """
+        :param baseline: Baseline nodes or topology version to set as baseline.
+        """
+        if isinstance(baseline, int):
+            result = self.__run("--baseline version %d --yes" % baseline)
+        else:
+            result = self.__run("--baseline set %s --yes" %
+                                ",".join([node.account.externally_routable_ip for node in baseline]))
+
+        return self.__parse_cluster_state(result)
+
+    def add_to_baseline(self, nodes):
+        """
+        :param nodes: Nodes that should be added to baseline.
+        """
+        result = self.__run("--baseline add %s --yes" %
+                            ",".join([node.account.externally_routable_ip for node in nodes]))
+
+        return self.__parse_cluster_state(result)
+
+    def remove_from_baseline(self, nodes):
+        """
+        :param nodes: Nodes that should be removed to baseline.
+        """
+        result = self.__run("--baseline remove %s --yes" %
+                            ",".join([node.account.externally_routable_ip for node in nodes]))
+
+        return self.__parse_cluster_state(result)
+
+    def disable_baseline_auto_adjust(self):
+        """
+        Disable baseline auto adjust.
+        """
+        return self.__run("--baseline auto_adjust disable --yes")
+
+    def enable_baseline_auto_adjust(self, timeout=None):
+        """
+        Enable baseline auto adjust.
+        :param timeout: Auto adjust timeout in millis.
+        """
+        timeout_str = "timeout %d" % timeout if timeout else ""
+        return self.__run("--baseline auto_adjust enable %s --yes" % timeout_str)
+
+    def activate(self):
+        """
+        Activate cluster.
+        """
+        return self.__run("--activate --yes")
+
+    def deactivate(self):
+        """
+        Deactivate cluster.
+        """
+        return self.__run("--deactivate --yes")
+
+    @staticmethod
+    def __parse_cluster_state(output):
+        state_pattern = re.compile("Cluster state: ([^\\s]+)")
+        topology_pattern = re.compile("Current topology version: (\\d+)")
+        baseline_pattern = re.compile("Consistent(Id|ID)=([^\\s]+),\\sS(tate|TATE)=([^\\s]+),?(\\sOrder=(\\d+))?")
+
+        match = state_pattern.search(output)
+        state = match.group(1) if match else None
+
+        match = topology_pattern.search(output)
+        topology = int(match.group(1)) if match else None
+
+        baseline = [BaselineNode(consistent_id=m[1], state=m[3], order=int(m[5]) if m[5] else None)
+                    for m in baseline_pattern.findall(output)]
+
+        return ClusterState(state=state, topology_version=topology, baseline=baseline)
+
+    def __run(self, cmd):
+        node = random.choice(self.__alives())
+
+        self.logger.debug("Run command %s on node %s", cmd, node.name)
+
+        raw_output = node.account.ssh_capture(self.__form_cmd(node, cmd), allow_fail=True)
+        code, output = self.__parse_output(raw_output)
+
+        self.logger.debug("Output of command %s on node %s, exited with code %d, is %s", cmd, node.name, code, output)
+
+        if code != 0:
+            raise ControlUtilityError(node.account, cmd, code, output)
+
+        return output
+
+    def __form_cmd(self, node, cmd):
+        return self._cluster.path.script("%s --host %s %s" % (self.BASE_COMMAND, node.account.externally_routable_ip,
+                                                              cmd))
+
+    @staticmethod
+    def __parse_output(raw_output):
+        exit_code = raw_output.channel_file.channel.recv_exit_status()
+        output = "".join(raw_output)
+
+        pattern = re.compile("Command \\[[^\\s]*\\] finished with code: (\\d+)")
+        match = pattern.search(output)
+
+        if match:
+            return int(match.group(1)), output
+        return exit_code, output
+
+    def __alives(self):
+        return [node for node in self._cluster.nodes if self._cluster.alive(node)]
+
+
+BaselineNode = namedtuple("BaselineNode", ["consistent_id", "state", "order"])
+ClusterState = namedtuple("ClusterState", ["state", "topology_version", "baseline"])
+
+
+class ControlUtilityError(RemoteCommandError):
+    """
+    Error is raised when control utility failed.
+    """
+    def __init__(self, account, cmd, exit_status, output):
+        super(ControlUtilityError, self).__init__(account, cmd, exit_status, "".join(output))

modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py

@@ -18,17 +18,19 @@
 """
 
 import os
-from abc import abstractmethod
+from abc import abstractmethod, ABCMeta
 
 from ducktape.services.background_thread import BackgroundThreadService
 from ducktape.utils.util import wait_until
+from six import add_metaclass
 
 from ignitetest.services.utils.ignite_config import IgniteLoggerConfig, IgniteServerConfig, IgniteClientConfig
 from ignitetest.services.utils.ignite_path import IgnitePath
 from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin
 from ignitetest.tests.utils.version import IgniteVersion
 
 
+@add_metaclass(ABCMeta)
 class IgniteAwareService(BackgroundThreadService):
     """
     The base class to build services aware of Ignite.
@@ -85,8 +87,14 @@ def init_persistent(self, node):
         logger_config = IgniteLoggerConfig().render(work_dir=self.WORK_DIR)
 
         node.account.mkdirs(self.PERSISTENT_ROOT)
-        node.account.create_file(self.CONFIG_FILE, self.config().render(
-            config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, properties=self.properties))
+
+        node_config = self.config().render(config_dir=self.PERSISTENT_ROOT,
+                                           work_dir=self.WORK_DIR,
+                                           properties=self.properties,
+                                           consistent_id=node.account.externally_routable_ip)
+
+        setattr(node, "consistent_id", node.account.externally_routable_ip)
+        node.account.create_file(self.CONFIG_FILE, node_config)
         node.account.create_file(self.LOG4J_CONFIG_FILE, logger_config)
 
     @abstractmethod
@@ -156,10 +164,9 @@ def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backof
         :param backoff_sec: Number of seconds to back off between each failure to meet the condition
                 before checking again.
         """
-        assert len(self.nodes) == 1
-
-        self.await_event_on_node(evt_message, self.nodes[0], timeout_sec, from_the_beginning=from_the_beginning,
-                                 backoff_sec=backoff_sec)
+        for node in self.nodes:
+            self.await_event_on_node(evt_message, node, timeout_sec, from_the_beginning=from_the_beginning,
+                                     backoff_sec=backoff_sec)
 
     def execute(self, command):
         """