Add JSON serialization test to the cluster state consistency check at…

… the end of the each integration test

src/test/java/org/elasticsearch/cluster/ClusterStateDiffTests.java

@@ -49,6 +49,8 @@
 
 import static org.elasticsearch.cluster.metadata.AliasMetaData.newAliasMetaDataBuilder;
 import static org.elasticsearch.test.ElasticsearchIntegrationTest.setRandomSettings;
+import static org.elasticsearch.common.xcontent.XContentTestUtils.convertToMap;
+import static org.elasticsearch.common.xcontent.XContentTestUtils.mapsEqualIgnoringArrayOrder;
 import static org.hamcrest.Matchers.equalTo;
 
 public class ClusterStateDiffTests extends ElasticsearchTestCase {
@@ -144,6 +146,9 @@ public void testClusterStateDiffSerialization() throws Exception {
                 assertThat(clusterStateFromDiffs.metaData().customs(), equalTo(clusterState.metaData().customs()));
                 assertThat(clusterStateFromDiffs.metaData().aliases(), equalTo(clusterState.metaData().aliases()));
 
+                // JSON Serialization test - make sure that both states produce similar JSON
+                assertThat(mapsEqualIgnoringArrayOrder(convertToMap(clusterStateFromDiffs), convertToMap(clusterState)), equalTo(true));
+
                 // Smoke test - we cannot compare bytes to bytes because some elements might get serialized in different order
                 // however, serialized size should remain the same
                 assertThat(ClusterState.Builder.toBytes(clusterStateFromDiffs).length, equalTo(ClusterState.Builder.toBytes(clusterState).length));

src/test/java/org/elasticsearch/common/xcontent/XContentTestUtils.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.Lists;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
+
+public final class XContentTestUtils {
+    private XContentTestUtils() {
+
+    }
+
+    public static Map<String, Object> convertToMap(ToXContent part) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        part.toXContent(builder, EMPTY_PARAMS);
+        builder.endObject();
+        return XContentHelper.convertToMap(builder.bytes(), false).v2();
+    }
+
+
+    /**
+     * Compares to maps generated from XContentObjects. The order of elements in arrays is ignored
+     */
+    public static boolean mapsEqualIgnoringArrayOrder(Map<String, Object> first, Map<String, Object> second) {
+        if (first.size() != second.size()) {
+            return false;
+        }
+
+        for (String key : first.keySet()) {
+            if (objectsEqualIgnoringArrayOrder(first.get(key), second.get(key)) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static boolean objectsEqualIgnoringArrayOrder(Object first, Object second) {
+        if (first == null ) {
+            return second == null;
+        } else if (first instanceof List) {
+            if (second instanceof  List) {
+                List<Object> secondList = Lists.newArrayList((List<Object>) second);
+                List<Object> firstList = (List<Object>) first;
+                if (firstList.size() == secondList.size()) {
+                    for (Object firstObj : firstList) {
+                        boolean found = false;
+                        for (Object secondObj : secondList) {
+                            if (objectsEqualIgnoringArrayOrder(firstObj, secondObj)) {
+                                secondList.remove(secondObj);
+                                found = true;
+                                break;
+                            }
+                        }
+                        if (found == false) {
+                            return false;
+                        }
+                    }
+                    return secondList.isEmpty();
+                } else {
+                    return false;
+                }
+            } else {
+                return false;
+            }
+        } else if (first instanceof Map) {
+            if (second instanceof Map) {
+                return mapsEqualIgnoringArrayOrder((Map<String, Object>) first, (Map<String, Object>) second);
+            } else {
+                return false;
+            }
+        } else {
+            return first.equals(second);
+        }
+    }
+
+}

src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.InconsistentClusterStateTest;
 import org.elasticsearch.test.InternalTestCluster;
 import org.junit.Test;
 
@@ -55,7 +56,8 @@
 /**
  *
  */
-@ClusterScope(scope= Scope.TEST, numDataNodes = 0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@InconsistentClusterStateTest(reason = "testShardActiveElseWhere might change the state of a non-master node")
 public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest {
 
     @Test

src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateTests.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.InconsistentClusterStateTest;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -54,6 +55,7 @@
 /**
  *
  */
+@InconsistentClusterStateTest(reason = "testBrokenMapping creates unserializable cluster state. It should be fixed in https://github.com/elastic/elasticsearch/pull/8802")
 public class SimpleIndexTemplateTests extends ElasticsearchIntegrationTest {
 
     @Test

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -138,6 +138,8 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
+import static org.elasticsearch.common.xcontent.XContentTestUtils.convertToMap;
+import static org.elasticsearch.common.xcontent.XContentTestUtils.mapsEqualIgnoringArrayOrder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.InternalTestCluster.clusterName;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;
@@ -1128,19 +1130,25 @@ void ensureClusterSizeConsistency() {
      * Verifies that all nodes that have the same version of the cluster state as master have same cluster state
      */
     void ensureClusterStateConsistency() throws IOException {
-        if (cluster() != null) {
+        if (cluster() != null && isInconsistentClusterStateTest(getTestClass()) == false) {
             ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState();
+            Map<String, Object> masterStateMap = convertToMap(masterClusterState);
             int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length;
             for (Client client : cluster()) {
                 ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState();
                 if (masterClusterState.version() == localClusterState.version()) {
-                    assertThat(masterClusterState.uuid(), equalTo(localClusterState.uuid()));
-                    // We cannot compare serialization bytes since serialization order of maps is not guaranteed
-                    // but we can compare serializaton sizes - they should be the same
-                    int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length;
-                    if(masterClusterStateSize != localClusterStateSize) {
+                    try {
+                        assertThat(masterClusterState.uuid(), equalTo(localClusterState.uuid()));
+                        // We cannot compare serialization bytes since serialization order of maps is not guaranteed
+                        // but we can compare serialization sizes - they should be the same
+                        int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length;
+                        assertThat(masterClusterStateSize, equalTo(localClusterStateSize));
+
+                        // Compare JSON serialization
+                        assertThat(mapsEqualIgnoringArrayOrder(masterStateMap, convertToMap(localClusterState)), equalTo(true));
+                    } catch (AssertionError error) {
                         logger.error("Cluster state from master:\n{}\nLocal cluster state:\n{}", masterClusterState.toString(), localClusterState.toString());
-                        fail("Master cluster state and local cluster state have diffrent sizes");
+                        throw error;
                     }
                 }
             }
@@ -1965,6 +1973,11 @@ private static boolean isSuiteScopedTest(Class<?> clazz) {
         return clazz.getAnnotation(SuiteScopeTest.class) != null;
     }
 
+
+    private static boolean isInconsistentClusterStateTest(Class<?> clazz) {
+        return clazz.getAnnotation(InconsistentClusterStateTest.class) != null;
+    }
+
     /**
      * If a test is annotated with {@link org.elasticsearch.test.ElasticsearchIntegrationTest.SuiteScopeTest}
      * the checks and modifications that are applied to the used test cluster are only done after all tests
@@ -1976,4 +1989,16 @@ private static boolean isSuiteScopedTest(Class<?> clazz) {
     @Ignore
     public @interface SuiteScopeTest {
     }
+
+    /**
+     * If a test is annotated with {@link org.elasticsearch.test.ElasticsearchIntegrationTest.InconsistentClusterStateTest}
+     * it might create inconsistent cluster state between nodes as part of the test. This annotation suppresses cluster state consistency checks.
+     */
+    @Retention(RetentionPolicy.RUNTIME)
+    @Inherited
+    @Ignore
+    public @interface InconsistentClusterStateTest {
+        String reason();
+    }
+
 }