IGNITE-12701 : Disallow silent deactivation in CLI and REST. (#7471)

* IGNITE-12597 : first impl

* IGNITE-12597 : + checking of reuse-data-region system property

* IGNITE-12597 : + fix in the JMX

* IGNITE-12597 : Think is ready except the JMX tests

* IGNITE-12597 : Think the ticket is done. Attempt #1.

* IGNITE-12597 : Small fix in messages.

* IGNITE-12597 : Fixed checking of erase-mem-on-deactivation-flag. Now within the job, not on client node.

* IGNITE-12597 : Fixed checking of erase-mem-on-deactivation-flag. Now within the job, not on client node.

* IGNITE-12597 : Lost fixws after review

* IGNITE-12597 : fix after review.

* deleteme

* IGNITE-12597 : fix after review #n.

* IGNITE-12597 : fix.

* IGNITE-12597 : Removed checking of reuse-mem-on-deactivate. Useless. Won't work.

* IGNITE-12597 : Minor fixes.

* IGNITE-12597 : Redeem IgniteMXBean

* IGNITE-12597 : code style

* IGNITE-12597 : fix force-change-state.

* IGNITE-12597 : fix.

* IGNITE-12597 : spell fix.

* IGNITE-12597 : removed exception throwing for the internal calls satbility.

* IGNITE-12646 : checking of deactivation returned

* IGNITE-12646 : Suggestion of fixing mbean interface IgniteMXBean.

* IGNITE-12646 : + some javadoc

* IGNITE-12646 : + enchanced javadoc

* IGNITE-12646 : changed IgniteCluster, Ignite, IgniteMXBean

* IGNITE-12646 : another exception

* IGNITE-12646 : fix of import

* IGNITE-12646 : first working version of
new IgniteCluster#state(ClusterState, boolean force)

* IGNITE-12614 : fixed REST, java tests.

* IGNITE-12701 : fix after review

* IGNITE-12701 : redeem.

* IGNITE-12701 : minor comment fit.

* IGNITE-12701 : minor fix.

* IGNITE-12701 : review fixes.

* IGNITE-12701 : back to checking of state change through the discovery.

* IGNITE-12701 : critical fix.

* IGNITE-12701 : codestyle fix.

* IGNITE-12701 : removed API change.

* IGNITE-12701 : minority.

* IGNITE-12701 : minority review fix.

* IGNITE-12701 : last review.

* IGNITE-12701 : fix resetting params in the commands.

* IGNITE-12701 : attempt to fix nodeJS thin client test

* IGNITE-12701 : minority.

* IGNITE-12701 : critical fix : checking the feature before deactivation.

* IGNITE-12701 : fixes after review.

* IGNITE-12701 : fixes after review.

* IGNITE-12701 : fixes after review 2.

* IGNITE-12701 : fixes after review 3.

* Minor javadoc update.

* Update GridClientClusterState.java

* Update GridClientClusterState.java

* Update IgniteCluster.java

* Update GridClientConnection.java

* IGNITE-12701 : fixes after review 4.

* IGNITE-12701 : fixes after review 5

* IGNITE-12701 : ChangeGlobalStateMessage is now forced by default.

* IGNITE-12701 : minor change of an error message.

* IGNITE-12701 : fixed after prereview 6

* IGNITE-12701 : reverted change global state methods.

* IGNITE-12701 : reverted proj settings.

* IGNITE-12701 : reverted proj settings.

* IGNITE-12701 : Fix of comments, spelling, @code True->true / false / null.

* IGNITE-12701 : Fix of comments, spelling, @code True->true / false / null 2.

* IGNITE-12701 : aalse -> false

* IGNITE-12701 : better 'fails if' description.

* IGNITE-12701 : better 'fails if' description 2.

* IGNITE-12701 : breakline.

* IGNITE-12701 : better see.

* IGNITE-12701 : better see.

* IGNITE-12701 : minorities.

* IGNITE-12701 : fixed notes of data loss.

* IGNITE-12703 : fixes after review.

* IGNITE-12703: Same comments.

* IGNITE-12703 + comments

* IGNITE-12703 : renaming, refactoring.

* IGNITE-12703 : reverted minority.

* IGNITE-12703 : major fix.

* IGNITE-12703 : + comment.

* IGNITE-12703 : + breakline.

* IGNITE-12703 : removed testInternalForcedDeactivation()

Co-authored-by: Vladimir Steshin <VASteshin@sberbank.ru>
Co-authored-by: Nikolay <nizhikov@apache.org>

modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java

@@ -143,8 +143,11 @@
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.cluster.ClusterState.ACTIVE;
+import static org.apache.ignite.cluster.ClusterState.INACTIVE;
 import static org.apache.ignite.configuration.WALMode.NONE;
 import static org.apache.ignite.internal.IgniteVersionUtils.VER_STR;
+import static org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor.DATA_LOST_ON_DEACTIVATION_WARNING;
 import static org.apache.ignite.internal.processors.query.QueryUtils.TEMPLATE_PARTITIONED;
 import static org.apache.ignite.internal.processors.query.QueryUtils.TEMPLATE_REPLICATED;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_FAILED;
@@ -254,18 +257,35 @@ private void assertCacheMetrics(String content) throws IOException {
      * @throws IOException If parsing failed.
      */
     protected JsonNode validateJsonResponse(String content) throws IOException {
+        return validateJsonResponse(content, false);
+    }
+
+    /**
+     * Validates JSON response.
+     *
+     * @param content Content to check.
+     * @param errorExpected is error expected.
+     * @return REST result if {@code errorExpected} is {@code false}. Error instead.
+     * @throws IOException If parsing failed.
+     */
+    protected JsonNode validateJsonResponse(String content, boolean errorExpected) throws IOException {
         assertNotNull(content);
         assertFalse(content.isEmpty());
 
         JsonNode node = JSON_MAPPER.readTree(content);
 
-        assertTrue("Unexpected error: " + node.get("error").asText(), node.get("error").isNull());
+        JsonNode errNode = node.get("error");
+
+        if (errorExpected)
+            assertTrue("Expected an error.", !errNode.isNull());
+        else
+            assertTrue("Unexpected error: " + errNode.asText(), errNode.isNull());
 
         assertEquals(STATUS_SUCCESS, node.get("successStatus").asInt());
 
         assertNotSame(securityEnabled(), node.get("sessionToken").isNull());
 
-        return node.get("response");
+        return node.get(errorExpected ? "error" : "response");
     }
 
     /**
@@ -887,11 +907,23 @@ public void testGetAndReplace() throws Exception {
     public void testDeactivateActivate() throws Exception {
         assertClusterState(true);
 
+        changeClusterState(GridRestCommand.CLUSTER_SET_STATE, "state", INACTIVE.name());
+
+        changeClusterState(GridRestCommand.CLUSTER_SET_STATE, "state", INACTIVE.name(), "force", "true");
+
+        changeClusterState(GridRestCommand.CLUSTER_SET_STATE, "state", ACTIVE.name());
+
         changeClusterState(GridRestCommand.CLUSTER_DEACTIVATE);
+
+        changeClusterState(GridRestCommand.CLUSTER_DEACTIVATE, "force", "true");
+
         changeClusterState(GridRestCommand.CLUSTER_ACTIVATE);
 
         // same for deprecated.
         changeClusterState(GridRestCommand.CLUSTER_INACTIVE);
+
+        changeClusterState(GridRestCommand.CLUSTER_INACTIVE, "force", "true");
+
         changeClusterState(GridRestCommand.CLUSTER_ACTIVE);
 
         initCache();
@@ -3181,16 +3213,37 @@ private void assertClusterState(boolean exp) throws Exception {
      * Change cluster state and test new state.
      *
      * @param cmd Command.
+     * @param params Arguments for {@code cmd}.
      * @throws Exception If failed.
      */
-    private void changeClusterState(GridRestCommand cmd) throws Exception {
-        String ret = content(null, cmd);
+    private void changeClusterState(GridRestCommand cmd, String... params) throws Exception {
+        String ret = content(null, cmd, params);
 
-        JsonNode res = validateJsonResponse(ret);
+        boolean force = false;
+
+        boolean deactivate = cmd == GridRestCommand.CLUSTER_INACTIVE || cmd == GridRestCommand.CLUSTER_DEACTIVATE;
+
+        for (int i = 0; i < params.length; ++i) {
+            String p = params[i];
+
+            if ("force".equals(p) && params[i + 1].equals("true"))
+                force = true;
+
+            if (cmd == GridRestCommand.CLUSTER_SET_STATE && p.equals("state"))
+                deactivate = params[i + 1].equals(INACTIVE.name());
+        }
+
+        boolean errorExpected = !force && deactivate;
+
+        JsonNode res = validateJsonResponse(ret, errorExpected);
 
         assertFalse(res.isNull());
-        assertTrue(res.asText().startsWith(cmd.key()));
 
-        assertClusterState(cmd == GridRestCommand.CLUSTER_ACTIVATE || cmd == GridRestCommand.CLUSTER_ACTIVE);
+        if (errorExpected)
+            assertTrue(res.asText().contains(DATA_LOST_ON_DEACTIVATION_WARNING));
+        else
+            assertTrue(res.asText().startsWith(cmd.key()));
+
+        assertClusterState(!deactivate || !force);
     }
 }

modules/core/src/main/java/org/apache/ignite/Ignite.java

@@ -663,6 +663,9 @@ public <T> IgniteQueue<T> queue(String name, int cap, @Nullable CollectionConfig
 
     /**
      * Changes Ignite grid state to active or inactive.
+     * <p>
+     * <b>NOTE:</b>
+     * Deactivation clears in-memory caches (without persistence) including the system caches.
      *
      * @param active If {@code True} start activation process. If {@code False} start deactivation process.
      * @throws IgniteException If there is an already started transaction or lock in the same thread.

modules/core/src/main/java/org/apache/ignite/IgniteCluster.java

@@ -455,6 +455,9 @@ public IgniteFuture<Collection<ClusterStartNodeResult>> startNodesAsync(Collecti
 
     /**
      * Changes Ignite grid state to active or inactive.
+     * <p>
+     * <b>NOTE:</b>
+     * Deactivation clears in-memory caches (without persistence) including the system caches.
      *
      * @param active If {@code True} start activation process. If {@code False} start deactivation process.
      * @throws IgniteException If there is an already started transaction or lock in the same thread.
@@ -472,6 +475,9 @@ public IgniteFuture<Collection<ClusterStartNodeResult>> startNodesAsync(Collecti
 
     /**
      * Changes current cluster state to given {@code newState} cluster state.
+     * <p>
+     * <b>NOTE:</b>
+     * Deactivation clears in-memory caches (without persistence) including the system caches.
      *
      * @param newState New cluster state.
      * @throws IgniteException If there is an already started transaction or lock in the same thread.

modules/core/src/main/java/org/apache/ignite/cluster/ClusterState.java

@@ -23,7 +23,12 @@
  * Cluster states.
  */
 public enum ClusterState {
-    /** Cluster deactivated. Cache operations aren't allowed. */
+    /**
+     * Cluster deactivated. Cache operations aren't allowed.
+     * <p>
+     * <b>NOTE:</b>
+     * Deactivation clears in-memory caches (without persistence) including the system caches.
+     */
     INACTIVE,
 
     /** Cluster activated. All cache operations are allowed. */

modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java

@@ -19,6 +19,7 @@
 
 import java.util.BitSet;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cluster.ClusterState;
 import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.communication.tcp.messages.HandshakeWaitMessage;
@@ -87,6 +88,13 @@ public enum IgniteFeatures {
     /** ContinuousQuery with security subject id support. */
     CONT_QRY_SECURITY_AWARE(21),
 
+    /**
+     * Preventing loss of in-memory data when deactivating the cluster.
+     *
+     * @see ClusterState#INACTIVE
+     */
+    SAFE_CLUSTER_DEACTIVATION(22),
+
     /** Long operations dump timeout. */
     LONG_OPERATIONS_DUMP_TIMEOUT(30);
 

modules/core/src/main/java/org/apache/ignite/internal/client/GridClientClusterState.java

@@ -25,14 +25,15 @@
 public interface GridClientClusterState {
     /**
      * @param active {@code True} activate, {@code False} deactivate.
-     * @deprecated Use {@link #state()} instead.
+     * @throws GridClientException If the request to change the cluster state failed.
+     * @deprecated Use {@link #state(ClusterState, boolean)} instead.
      */
     @Deprecated
     public void active(boolean active) throws GridClientException;
 
     /**
      * @return {@code Boolean} - Current cluster state. {@code True} active, {@code False} inactive.
-     * @deprecated Use {@link #state(ClusterState)} instead.
+     * @deprecated Use {@link #state()} instead.
      */
     @Deprecated
     public boolean active() throws GridClientException;
@@ -47,9 +48,11 @@ public interface GridClientClusterState {
      * Changes cluster state to {@code newState}.
      *
      * @param newState New cluster state.
+     * @param forceDeactivation If {@code true}, cluster deactivation will be forced.
      * @throws GridClientException If the request to change the cluster state failed.
+     * @see ClusterState#INACTIVE
      */
-    public void state(ClusterState newState) throws GridClientException;
+    public void state(ClusterState newState, boolean forceDeactivation) throws GridClientException;
 
     /**
      * Get the cluster name.

modules/core/src/main/java/org/apache/ignite/internal/client/impl/GridClientClusterStateImpl.java

@@ -18,14 +18,20 @@
 package org.apache.ignite.internal.client.impl;
 
 import java.util.Collection;
+import java.util.UUID;
 import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.internal.IgniteFeatures;
 import org.apache.ignite.internal.client.GridClientClusterState;
 import org.apache.ignite.internal.client.GridClientException;
 import org.apache.ignite.internal.client.GridClientNode;
 import org.apache.ignite.internal.client.GridClientPredicate;
 import org.apache.ignite.internal.client.balancer.GridClientLoadBalancer;
 import org.apache.ignite.internal.client.impl.connection.GridClientConnection;
 
+import static org.apache.ignite.cluster.ClusterState.ACTIVE;
+import static org.apache.ignite.cluster.ClusterState.INACTIVE;
+import static org.apache.ignite.internal.client.util.GridClientUtils.checkFeatureSupportedByCluster;
+
 /**
  *
  */
@@ -50,7 +56,7 @@ public GridClientClusterStateImpl(
 
     /** {@inheritDoc} */
     @Override public void active(final boolean active) throws GridClientException {
-        withReconnectHandling((conn, nodeId) -> conn.changeState(active, nodeId)).get();
+        state(active ? ACTIVE : INACTIVE, true);
     }
 
     /** {@inheritDoc} */
@@ -64,8 +70,22 @@ public GridClientClusterStateImpl(
     }
 
     /** {@inheritDoc} */
-    @Override public void state(ClusterState newState) throws GridClientException {
-        withReconnectHandling((con, nodeId) -> con.changeState(newState, nodeId)).get();
+    @Override public void state(ClusterState newState, boolean forceDeactivation) throws GridClientException {
+        // Check compatibility of new forced deactivation on all nodes.
+        UUID oldVerNode = checkFeatureSupportedByCluster(client, IgniteFeatures.SAFE_CLUSTER_DEACTIVATION, false,
+            false);
+
+        if (oldVerNode == null)
+            withReconnectHandling((con, nodeId) -> con.changeState(newState, nodeId, forceDeactivation)).get();
+        else {
+            if (newState == INACTIVE && !forceDeactivation) {
+                throw new GridClientException("Deactivation stopped. Found a node not supporting checking of " +
+                    "safety of this operation: " + oldVerNode + ". Deactivation clears in-memory caches (without " +
+                    "persistence) including the system caches. To deactivate cluster pass flag 'force'.");
+            }
+
+            withReconnectHandling((con, nodeId) -> con.changeState(newState, nodeId)).get();
+        }
     }
 
     /** {@inheritDoc} */

modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java

@@ -25,7 +25,6 @@
 import java.util.Set;
 import java.util.UUID;
 import javax.net.ssl.SSLContext;
-
 import org.apache.ignite.cluster.ClusterState;
 import org.apache.ignite.internal.client.GridClientCacheFlag;
 import org.apache.ignite.internal.client.GridClientClosedException;
@@ -310,25 +309,26 @@ public abstract <R> GridClientFutureAdapter<R> execute(String taskName, Object a
         boolean keepBinaries) throws GridClientConnectionResetException, GridClientClosedException;
 
     /**
-     * Change grid global state.
+     * Changes grid global state.
      *
-     * @param active Active.
+     * @param state New cluster state.
      * @param destNodeId Destination node id.
-     * @deprecated Use {@link #changeState(ClusterState, UUID)} instead.
+     * @throws GridClientConnectionResetException In case of error.
+     * @throws GridClientClosedException If client was manually closed before request was sent over network.
      */
-    @Deprecated
-    public abstract GridClientFuture<?> changeState(boolean active, UUID destNodeId)
-            throws GridClientClosedException, GridClientConnectionResetException;
+    public abstract GridClientFuture<?> changeState(ClusterState state, UUID destNodeId)
+        throws GridClientClosedException, GridClientConnectionResetException;
 
     /**
      * Changes grid global state.
      *
      * @param state New cluster state.
      * @param destNodeId Destination node id.
+     * @param forceDeactivation If {@code true}, cluster deactivation will be forced.
      * @throws GridClientConnectionResetException In case of error.
      * @throws GridClientClosedException If client was manually closed before request was sent over network.
      */
-    public abstract GridClientFuture<?> changeState(ClusterState state, UUID destNodeId)
+    public abstract GridClientFuture<?> changeState(ClusterState state, UUID destNodeId, boolean forceDeactivation)
         throws GridClientClosedException, GridClientConnectionResetException;
 
     /**

modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java

@@ -62,6 +62,7 @@
 import org.apache.ignite.internal.processors.rest.client.message.GridClientCacheRequest;
 import org.apache.ignite.internal.processors.rest.client.message.GridClientClusterNameRequest;
 import org.apache.ignite.internal.processors.rest.client.message.GridClientClusterStateRequest;
+import org.apache.ignite.internal.processors.rest.client.message.GridClientClusterStateRequestV2;
 import org.apache.ignite.internal.processors.rest.client.message.GridClientHandshakeRequest;
 import org.apache.ignite.internal.processors.rest.client.message.GridClientMessage;
 import org.apache.ignite.internal.processors.rest.client.message.GridClientNodeBean;
@@ -84,8 +85,6 @@
 import org.jetbrains.annotations.Nullable;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.apache.ignite.cluster.ClusterState.ACTIVE;
-import static org.apache.ignite.cluster.ClusterState.INACTIVE;
 import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES;
 import static org.apache.ignite.internal.client.GridClientCacheFlag.encodeCacheFlags;
 import static org.apache.ignite.internal.client.impl.connection.GridClientConnectionCloseReason.CONN_IDLE;
@@ -817,11 +816,11 @@ private GridClientAuthenticationRequest buildAuthRequest() {
     }
 
     /** {@inheritDoc} */
-    @Override public GridClientFuture<?> changeState(
-        boolean active,
-        UUID destNodeId
-    ) throws GridClientClosedException, GridClientConnectionResetException {
-        return changeState(active ? ACTIVE : INACTIVE, destNodeId);
+    @Override public GridClientFuture<?> changeState(ClusterState state, UUID destNodeId, boolean forceDeactivation)
+        throws GridClientClosedException, GridClientConnectionResetException {
+        assert state != null;
+
+        return makeRequest(GridClientClusterStateRequestV2.state(state, forceDeactivation), destNodeId);
     }
 
     /** {@inheritDoc} */