IGNITE-13660 Unexpected NODE_LEFT on graceful stop (at ducktests)

modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/start_stop_client/IgniteCachePutClient.java → modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/client_test/IgniteCachePutClient.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.ducktest.tests.start_stop_client;
+package org.apache.ignite.internal.ducktest.tests.client_test;
 
 import java.util.Optional;
 import java.util.UUID;
@@ -33,7 +33,7 @@ public class IgniteCachePutClient extends IgniteAwareApplication {
 
         long pacing = Optional.ofNullable(jsonNode.get("pacing"))
                 .map(JsonNode::asLong)
-                .orElse(0l);
+                .orElse(0L);
 
         log.info("Test props:" +
                 " cacheName=" + cacheName +

modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/self_test/TestKillableApplication.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.self_test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class TestKillableApplication extends IgniteAwareApplication {
+    /** {@inheritDoc} */
+    @Override public void run(JsonNode jsonNode) throws Exception {
+        markInitialized();
+
+        while (!terminated())
+            U.sleep(100);
+
+        U.sleep(5000);
+
+        markFinished();
+    }
+}

modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/self_test/TestSelfKillableApplication.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.self_test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class TestSelfKillableApplication extends IgniteAwareApplication {
+    /** {@inheritDoc} */
+    @Override public void run(JsonNode jsonNode) throws Exception {
+        markInitialized();
+
+        U.sleep(5000);
+
+        markFinished();
+    }
+}

modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java

@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
+import sun.misc.Signal;
 
 /**
  *
@@ -73,7 +74,7 @@ public abstract class IgniteAwareApplication {
      * Default constructor.
      */
     protected IgniteAwareApplication() {
-        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+        Signal.handle(new Signal("TERM"), signal -> {
             log.info("SIGTERM recorded.");
 
             if (!finished && !broken)
@@ -100,11 +101,10 @@ protected IgniteAwareApplication() {
                 }
             }
 
-            if (log.isDebugEnabled())
-                log.debug("Graceful termination done.");
-        }));
+            log.info("Application finished. Waiting for graceful termination.");
+        });
 
-        log.info("ShutdownHook registered.");
+        log.info("SIGTERM handler registered.");
     }
 
     /**

modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py

@@ -186,6 +186,7 @@ def __init__(self, context, modules, servicejava_class_name, java_class_name, pa
 
         self.jvm_opts.extend([
             "-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file",
+            "-Dlog4j.configuration=file:" + self.LOG4J_CONFIG_FILE,
             "-Dlog4j.configDebug=true",
             "-DIGNITE_NO_SHUTDOWN_HOOK=true",  # allows to perform operations on app termination.
             "-Xmx1G",

modules/ducktests/tests/ignitetest/tests/client_in_out_test.py → modules/ducktests/tests/ignitetest/tests/client_test.py

@@ -18,14 +18,14 @@
 """
 import time
 
+from ducktape.mark import parametrize
 from ducktape.mark.resource import cluster
 
-from ducktape.mark import parametrize
 from ignitetest.services.ignite import IgniteService
 from ignitetest.services.ignite_app import IgniteApplicationService
 from ignitetest.services.utils.control_utility import ControlUtility
-from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration
 from ignitetest.services.utils.ignite_configuration import IgniteConfiguration
+from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration
 from ignitetest.utils import ignite_versions
 from ignitetest.utils.ignite_test import IgniteTest
 from ignitetest.utils.version import DEV_BRANCH, V_2_8_1, IgniteVersion
@@ -46,35 +46,27 @@ class ClientTest(IgniteTest):
 
     CACHE_NAME = "simple-tx-cache"
     PACING = 10
-    JAVA_CLIENT_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.start_stop_client.IgniteCachePutClient"
+    JAVA_CLIENT_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.client_test.IgniteCachePutClient"
 
+    # pylint: disable=R0913
     @ignite_versions(str(V_2_8_1), str(DEV_BRANCH))
     @cluster(num_nodes=7)
-    @parametrize(num_nodes=7,
-                 static_clients=2,
-                 temp_client=3,
-                 iteration_count=3,
-                 client_work_time=30)
-    # pylint: disable=R0913
-    def test_ignite_start_stop_nodes(self, ignite_version,
-                                     num_nodes, static_clients, temp_client, iteration_count, client_work_time):
+    @parametrize(num_nodes=7, static_clients=2, temp_client=3, iteration_count=3, client_work_time=30)
+    def test_ignite_start_stop_nodes(self, ignite_version, num_nodes, static_clients, temp_client, iteration_count,
+                                     client_work_time):
         """
         Start and stop clients node test without kill java process.
         Check topology.
         """
         self.ignite_start_stop(ignite_version, True, num_nodes, static_clients,
                                temp_client, iteration_count, client_work_time)
 
+    # pylint: disable=R0913
     @ignite_versions(str(V_2_8_1), str(DEV_BRANCH))
     @cluster(num_nodes=7)
-    @parametrize(num_nodes=7,
-                 static_clients=2,
-                 temp_client=3,
-                 iteration_count=3,
-                 client_work_time=30)
-    # pylint: disable=R0913
-    def test_ignite_kill_start_nodes(self, ignite_version,
-                                     num_nodes, static_clients, temp_client, iteration_count, client_work_time):
+    @parametrize(num_nodes=7, static_clients=2, temp_client=3, iteration_count=3, client_work_time=30)
+    def test_ignite_kill_start_nodes(self, ignite_version, num_nodes, static_clients, temp_client, iteration_count,
+                                     client_work_time):
         """
         Start and kill client nodes, Check topology
         """
@@ -83,69 +75,66 @@ def test_ignite_kill_start_nodes(self, ignite_version,
 
     # pylint: disable=R0914
     # pylint: disable=R0913
-    def ignite_start_stop(self, ignite_version, correct_stop_temp_node,
-                          nodes_num, static_clients_num, temp_client, iteration_count, client_work_time):
+    def ignite_start_stop(self, ignite_version, correct_stop_temp_node, nodes_num, static_clients_num, temp_client,
+                          iteration_count, client_work_time):
         """
         Test for starting and stopping fat clients.
         """
 
         servers_count = nodes_num - static_clients_num - temp_client
-
         current_top_v = servers_count
+
         # Topology version after test.
         fin_top_ver = servers_count + (2 * static_clients_num) + (2 * iteration_count * temp_client)
 
-        server_cfg = IgniteConfiguration(
-            version=IgniteVersion(ignite_version),
-            caches=[CacheConfiguration(name=self.CACHE_NAME, backups=1, atomicity_mode='TRANSACTIONAL')]
-        )
+        server_cfg = IgniteConfiguration(version=IgniteVersion(ignite_version), caches=[
+            CacheConfiguration(name=self.CACHE_NAME, backups=1, atomicity_mode='TRANSACTIONAL')])
 
         ignite = IgniteService(self.test_context, server_cfg, num_nodes=servers_count)
+
         control_utility = ControlUtility(ignite, self.test_context)
 
         client_cfg = server_cfg._replace(client_mode=True)
 
-        static_clients = IgniteApplicationService(
-            self.test_context,
-            client_cfg,
-            java_class_name=self.JAVA_CLIENT_CLASS_NAME,
-            num_nodes=static_clients_num,
-            params={"cacheName": self.CACHE_NAME,
-                    "pacing": self.PACING})
-
-        temp_clients = IgniteApplicationService(
-            self.test_context,
-            client_cfg,
-            java_class_name=self.JAVA_CLIENT_CLASS_NAME,
-            num_nodes=temp_client,
-            params={"cacheName": self.CACHE_NAME,
-                    "pacing": self.PACING})
+        static_clients = IgniteApplicationService(self.test_context, client_cfg,
+                                                  java_class_name=self.JAVA_CLIENT_CLASS_NAME,
+                                                  num_nodes=static_clients_num,
+                                                  params={"cacheName": self.CACHE_NAME, "pacing": self.PACING})
+
+        temp_clients = IgniteApplicationService(self.test_context, client_cfg,
+                                                java_class_name=self.JAVA_CLIENT_CLASS_NAME, num_nodes=temp_client,
+                                                params={"cacheName": self.CACHE_NAME, "pacing": self.PACING})
 
         ignite.start()
 
         static_clients.start()
 
         current_top_v += static_clients_num
+
         check_topology(control_utility, current_top_v)
 
         # Start / stop temp_clients node. Check cluster.
         for i in range(iteration_count):
-            self.logger.debug(f'Starting iteration: {i}.')
+            self.logger.info(f'Starting iteration: {i}.')
 
             temp_clients.start()
+
             current_top_v += temp_client
 
             await_event(static_clients, f'ver={current_top_v}, locNode=')
+
             check_topology(control_utility, current_top_v)
 
             await_event(temp_clients, f'clients={static_clients_num + temp_client}')
 
             time.sleep(client_work_time)
+
             temp_clients.stop(correct_stop_temp_node)
 
             current_top_v += temp_client
 
         await_event(static_clients, f'ver={current_top_v}, locNode=')
+
         static_clients.stop()
 
         check_topology(control_utility, fin_top_ver)

modules/ducktests/tests/ignitetest/tests/assertion_test.py → modules/ducktests/tests/ignitetest/tests/self_test.py

@@ -19,16 +19,18 @@
 
 from ducktape.mark.resource import cluster
 
+from ignitetest.services.ignite import IgniteService
 from ignitetest.services.ignite_app import IgniteApplicationService
 from ignitetest.services.ignite_execution_exception import IgniteExecutionException
-from ignitetest.services.utils.ignite_configuration import IgniteConfiguration
+from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteClientConfiguration
+from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster
 from ignitetest.utils import ignite_versions
 from ignitetest.utils.ignite_test import IgniteTest
 from ignitetest.utils.version import DEV_BRANCH, IgniteVersion
 
 
 # pylint: disable=W0223
-class SmokeSelfTest(IgniteTest):
+class SelfTest(IgniteTest):
     """
     Self test
     """
@@ -53,3 +55,43 @@ def test_assertion_convertion(self, ignite_version):
         else:
             app.stop()
             assert False
+
+    @cluster(num_nodes=4)
+    @ignite_versions(str(DEV_BRANCH))
+    def test_simple_services_start_stop(self, ignite_version):
+        """
+        Tests plain services start and stop (termitation vs self-terination).
+        """
+        ignites = IgniteService(self.test_context, IgniteConfiguration(version=IgniteVersion(ignite_version)),
+                                num_nodes=1)
+
+        ignites.start()
+
+        client = IgniteService(self.test_context, IgniteClientConfiguration(version=IgniteVersion(ignite_version)),
+                               num_nodes=1)
+
+        client.start()
+
+        node1 = IgniteApplicationService(
+            self.test_context,
+            IgniteClientConfiguration(version=IgniteVersion(ignite_version),
+                                      discovery_spi=from_ignite_cluster(ignites)),
+            java_class_name="org.apache.ignite.internal.ducktest.tests.self_test.TestKillableApplication",
+            timeout_sec=180)
+
+        node2 = IgniteApplicationService(
+            self.test_context,
+            IgniteClientConfiguration(version=IgniteVersion(ignite_version),
+                                      discovery_spi=from_ignite_cluster(ignites)),
+            java_class_name="org.apache.ignite.internal.ducktest.tests.self_test.TestSelfKillableApplication",
+            timeout_sec=180)
+
+        node1.start()
+
+        node2.run()
+
+        node1.stop()
+
+        client.stop()
+
+        ignites.stop()

modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml

@@ -29,5 +29,5 @@ rebalance:
   - ../add_node_rebalance_test.py
 
 clients:
-  - ../client_in_out_test.py
+  - ../client_test.py