_all: report conflict on merge and throw exception on doc_values

- _all field was never merged when mapping was updated and no conflict reported
- _all accepted doc_values format although it is always tokenized

relates to #777
closes #7377

src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
+import org.elasticsearch.index.mapper.object.RootObjectMapper;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.similarity.SimilarityLookupService;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -150,6 +151,9 @@ protected AllFieldMapper(String name, FieldType fieldType, NamedAnalyzer indexAn
                              @Nullable Settings fieldDataSettings, Settings indexSettings) {
         super(new Names(name, name, name, name), 1.0f, fieldType, null, indexAnalyzer, searchAnalyzer, postingsProvider, docValuesProvider,
                 similarity, normsLoading, fieldDataSettings, indexSettings);
+        if (hasDocValues()) {
+            throw new MapperParsingException("Field [" + names.fullName() + "] is always tokenized and cannot have doc values");
+        }
         this.enabled = enabled;
         this.autoBoost = autoBoost;
 
@@ -341,15 +345,12 @@ private void innerToXContent(XContentBuilder builder, boolean includeDefaults) t
         }
     }
 
-
     @Override
     public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
-        // do nothing here, no merging, but also no exception
-    }
-
-    @Override
-    public boolean hasDocValues() {
-        return false;
+        if (((AllFieldMapper)mergeWith).enabled() != this.enabled()) {
+            mergeContext.addConflict("mapper [" + names.fullName() + "] enabled is " + this.enabled() + " now encountering "+ ((AllFieldMapper)mergeWith).enabled());
+        }
+        super.merge(mergeWith, mergeContext);
     }
 
     @Override

src/test/java/org/elasticsearch/index/mapper/all/AllMapperOnCusterTests.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.all;
+
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.junit.Test;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.containsString;
+
+public class AllMapperOnCusterTests extends ElasticsearchIntegrationTest {
+
+    private static final String INDEX = "index";
+    private static final String TYPE = "type";
+
+    @Test
+    public void test_doc_valuesInvalidMapping() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").startObject("fielddata").field("format", "doc_values").endObject().endObject().endObject().endObject().endObject().string();
+        try {
+            prepareCreate(INDEX).setSource(mapping).get();
+            fail();
+        } catch (MapperParsingException e) {
+            assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values"));
+        }
+    }
+}

src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnCusterTests.java

@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.update;
+
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MergeMappingException;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.junit.Test;
+
+import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+
+public class UpdateMappingOnCusterTests extends ElasticsearchIntegrationTest {
+
+    private static final String INDEX = "index";
+    private static final String TYPE = "type";
+
+
+    @Test
+    public void test_all_enabled() throws Exception {
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").field("enabled", "false").endObject().endObject().endObject().endObject();
+        XContentBuilder mappingUpdate = jsonBuilder().startObject().startObject("_all").field("enabled", "true").endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject();
+        String errorMessage = "[_all] enabled is false now encountering true";
+        testConflict(mapping.string(), mappingUpdate.string(), errorMessage);
+    }
+
+    @Test
+    public void test_all_conflicts() throws Exception {
+        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_create_index.json");
+        String mappingUpdate = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json");
+        String[] errorMessage = {"[_all] enabled is true now encountering false",
+                "[_all] cannot enable norms (`norms.enabled`)",
+                "[_all] has different store values",
+                "[_all] has different store_term_vector values",
+                "[_all] has different store_term_vector_offsets values",
+                "[_all] has different store_term_vector_positions values",
+                "[_all] has different store_term_vector_payloads values",
+                "[_all] has different index_analyzer",
+                "[_all] has different similarity"};
+        // auto_boost and fielddata and search_analyzer should not report conflict
+        testConflict(mapping, mappingUpdate, errorMessage);
+    }
+
+    @Test
+    public void test_doc_valuesInvalidMapping() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").startObject("fielddata").field("format", "doc_values").endObject().endObject().endObject().endObject().endObject().string();
+        try {
+            prepareCreate(INDEX).setSource(mapping).get();
+            fail();
+        } catch (MapperParsingException e) {
+            assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values"));
+        }
+    }
+
+    @Test
+    public void test_doc_valuesInvalidMappingOnUpdate() throws Exception {
+        String mapping = jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject().string();
+        prepareCreate(INDEX).addMapping(TYPE, mapping).get();
+        String mappingUpdate = jsonBuilder().startObject().startObject(TYPE).startObject("_all").startObject("fielddata").field("format", "doc_values").endObject().endObject().endObject().endObject().string();
+        GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
+        try {
+            client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mappingUpdate).get();
+            fail();
+        } catch (MapperParsingException e) {
+            assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values"));
+        }
+        // make sure all nodes have same cluster state
+        compareMappingOnNodes(mappingsBeforeUpdateResponse);
+    }
+
+    protected void testConflict(String mapping, String mappingUpdate, String... errorMessages) throws InterruptedException {
+        assertAcked(prepareCreate(INDEX).setSource(mapping).get());
+        ensureGreen(INDEX);
+        GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
+        try {
+            client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mappingUpdate).get();
+            fail();
+        } catch (MergeMappingException e) {
+            for (String errorMessage : errorMessages) {
+                assertThat(e.getDetailedMessage(), containsString(errorMessage));
+            }
+        }
+        compareMappingOnNodes(mappingsBeforeUpdateResponse);
+
+    }
+
+    private void compareMappingOnNodes(GetMappingsResponse mappingsBeforeUpdateResponse) {
+        // make sure all nodes have same cluster state
+        for (Client client : cluster()) {
+            GetMappingsResponse mappingsAfterUpdateResponse = client.admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).setLocal(true).get();
+            assertThat(mappingsBeforeUpdateResponse.getMappings().get(INDEX).get(TYPE).source(), equalTo(mappingsAfterUpdateResponse.getMappings().get(INDEX).get(TYPE).source()));
+        }
+    }
+}

src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java

@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.update;
+
+import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.service.IndexService;
+import org.elasticsearch.test.ElasticsearchSingleNodeTest;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+
+public class UpdateMappingTests extends ElasticsearchSingleNodeTest {
+
+    @Test
+    public void test_all_enabled_after_disabled() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().endObject();
+        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject();
+        testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
+    }
+
+    @Test
+    public void test_all_disabled_after_enabled() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().endObject();
+        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject();
+        testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
+    }
+
+    @Test
+    public void test_all_disabled_after_default_enabled() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("some_text").field("type", "string").endObject().endObject().endObject();
+        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject();
+        testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
+    }
+
+    @Test
+    public void test_all_enabled_after_enabled() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().endObject();
+        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject();
+        XContentBuilder expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject().endObject();
+        testNoConflictWhileMergingAndMappingChanged(mapping, mappingUpdate, expectedMapping);
+    }
+
+    @Test
+    public void test_all_disabled_after_disabled() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().endObject();
+        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject();
+        XContentBuilder expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject().endObject();
+        testNoConflictWhileMergingAndMappingChanged(mapping, mappingUpdate, expectedMapping);
+    }
+
+    private void testNoConflictWhileMergingAndMappingChanged(XContentBuilder mapping, XContentBuilder mappingUpdate, XContentBuilder expectedMapping) throws IOException {
+        IndexService indexService = createIndex("test", ImmutableSettings.settingsBuilder().build(), "type", mapping);
+        // simulate like in MetaDataMappingService#putMapping
+        DocumentMapper.MergeResult mergeResult = indexService.mapperService().documentMapper("type").merge(indexService.mapperService().parse("type", new CompressedString(mappingUpdate.bytes()), true), DocumentMapper.MergeFlags.mergeFlags().simulate(false));
+        // assure we have no conflicts
+        assertThat(mergeResult.conflicts().length, equalTo(0));
+        // make sure mappings applied
+        CompressedString mappingAfterUpdate = indexService.mapperService().documentMapper("type").mappingSource();
+        assertThat(mappingAfterUpdate.toString(), equalTo(expectedMapping.string()));
+    }
+
+    public void testConflictFieldsMapping(String fieldName) throws Exception {
+        //test store, ... all the parameters that are not to be changed just like in other fields
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject(fieldName).field("enabled", true).field("store", "no").endObject()
+                .endObject().endObject();
+        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject(fieldName).field("enabled", true).field("store", "yes").endObject()
+                .startObject("properties").startObject("text").field("type", "string").endObject().endObject()
+                .endObject().endObject();
+        testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
+    }
+
+    protected void testConflictWhileMergingAndMappingUnchanged(XContentBuilder mapping, XContentBuilder mappingUpdate) throws IOException {
+        IndexService indexService = createIndex("test", ImmutableSettings.settingsBuilder().build(), "type", mapping);
+        CompressedString mappingBeforeUpdate = indexService.mapperService().documentMapper("type").mappingSource();
+        // simulate like in MetaDataMappingService#putMapping
+        DocumentMapper.MergeResult mergeResult = indexService.mapperService().documentMapper("type").merge(indexService.mapperService().parse("type", new CompressedString(mappingUpdate.bytes()), true), DocumentMapper.MergeFlags.mergeFlags().simulate(true));
+        // assure we have conflicts
+        assertThat(mergeResult.conflicts().length, equalTo(1));
+        // make sure simulate flag actually worked - no mappings applied
+        CompressedString mappingAfterUpdate = indexService.mapperService().documentMapper("type").mappingSource();
+        assertThat(mappingAfterUpdate, equalTo(mappingBeforeUpdate));
+    }
+}

src/test/java/org/elasticsearch/index/mapper/update/all_mapping_create_index.json

@@ -0,0 +1,32 @@
+{
+  "mappings": {
+    "type": {
+      "_all": {
+        "auto_boost": true,
+        "store": true,
+        "store_term_vectors": true,
+        "store_term_vector_offsets": true,
+        "store_term_vector_positions": true,
+        "store_term_vector_payloads": true,
+        "omit_norms": true,
+        "index_analyzer": "standard",
+        "search_analyzer": "whitespace",
+        "similarity": "my_similarity",
+        "fielddata": {
+           "format": "fst"
+        }
+      }
+    }
+  },
+  "settings": {
+    "similarity": {
+      "my_similarity": {
+        "type": "DFR",
+        "basic_model": "g",
+        "after_effect": "l",
+        "normalization": "h2",
+        "normalization.h2.c": "3.0"
+      }
+    }
+  }
+}

src/test/java/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json

@@ -0,0 +1,20 @@
+{
+  "type": {
+    "_all": {
+      "auto_boost": false,
+      "store": false,
+      "enabled": false,
+      "store_term_vectors": false,
+      "store_term_vector_offsets": false,
+      "store_term_vector_positions": false,
+      "store_term_vector_payloads": false,
+      "omit_norms": false,
+      "index_analyzer": "whitespace",
+      "search_analyzer": "standard",
+      "similarity": "bm25",
+      "fielddata": {
+          "format": "paged_bytes"
+      }
+    }
+  }
+}