diff --git a/docs/reference/index-modules/fielddata.asciidoc b/docs/reference/index-modules/fielddata.asciidoc index d74d648700f18..01f094b86acf6 100644 --- a/docs/reference/index-modules/fielddata.asciidoc +++ b/docs/reference/index-modules/fielddata.asciidoc @@ -26,6 +26,8 @@ example, can be set to `5m` for a 5 minute expiry. === Field data formats +The field data format controls how field data should be stored. + Depending on the field type, there might be several field data types available. In particular, string and numeric types support the `doc_values` format which allows for computing the field data data-structures at indexing @@ -33,6 +35,9 @@ time and storing them on disk. Although it will make the index larger and may be slightly slower, this implementation will be more near-realtime-friendly and will require much less memory from the JVM than other implementations. +Here is an example of how to configure the `tag` field to use the `fst` field +data format. + [source,js] -------------------------------------------------- { @@ -45,6 +50,25 @@ and will require much less memory from the JVM than other implementations. } -------------------------------------------------- +It is possible to change the field data format (and the field data settings +in general) on a live index by using the update mapping API. When doing so, +field data which had already been loaded for existing segments will remain +alive while new segments will use the new field data configuration. Thanks to +the background merging process, all segments will eventually use the new +field data format. + +[float] +==== Disallowing field data loading + +Field data can take a lot of RAM so it makes sense to disable field data +loading on the fields that don't need field data, for example those that are +used for full-text search only. In order to disable field data loading, just +change the field data type to `disabled`. Request that will try to load field +data on any field which is configured with this format will then return an +error. + +The `disabled` format is supported by all field types. + [float] ==== String field data types diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index cf10906c7d90d..b8284c2e92256 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import org.apache.lucene.index.IndexReader; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.collect.MapBuilder; @@ -36,12 +37,19 @@ import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import java.util.Map; import java.util.concurrent.ConcurrentMap; /** */ public class IndexFieldDataService extends AbstractIndexComponent { + private static final String DISABLED_FORMAT = "disabled"; + private static final String DOC_VALUES_FORMAT = "doc_values"; + private static final String ARRAY_FORMAT = "array"; + private static final String PAGED_BYTES_FORMAT = "paged_bytes"; + private static final String FST_FORMAT = "fst"; + private final static ImmutableMap buildersByType; private final static ImmutableMap docValuesBuildersByType; private final static ImmutableMap, IndexFieldData.Builder> buildersByTypeAndFormat; @@ -69,30 +77,47 @@ public class IndexFieldDataService extends AbstractIndexComponent { .immutableMap(); buildersByTypeAndFormat = MapBuilder., IndexFieldData.Builder>newMapBuilder() - .put(Tuple.tuple("string", "paged_bytes"), new PagedBytesIndexFieldData.Builder()) - .put(Tuple.tuple("string", "fst"), new FSTBytesIndexFieldData.Builder()) - .put(Tuple.tuple("string", "doc_values"), new DocValuesIndexFieldData.Builder()) - .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder()) - .put(Tuple.tuple("float", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FLOAT)) - .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder()) - .put(Tuple.tuple("double", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.DOUBLE)) - .put(Tuple.tuple("byte", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.BYTE)) - .put(Tuple.tuple("byte", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BYTE)) - .put(Tuple.tuple("short", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.SHORT)) - .put(Tuple.tuple("short", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT)) - .put(Tuple.tuple("int", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.INT)) - .put(Tuple.tuple("int", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT)) - .put(Tuple.tuple("long", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.LONG)) - .put(Tuple.tuple("long", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG)) - .put(Tuple.tuple("geo_point", "array"), new GeoPointDoubleArrayIndexFieldData.Builder()) + .put(Tuple.tuple("string", PAGED_BYTES_FORMAT), new PagedBytesIndexFieldData.Builder()) + .put(Tuple.tuple("string", FST_FORMAT), new FSTBytesIndexFieldData.Builder()) + .put(Tuple.tuple("string", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder()) + .put(Tuple.tuple("string", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + + .put(Tuple.tuple("float", ARRAY_FORMAT), new FloatArrayIndexFieldData.Builder()) + .put(Tuple.tuple("float", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FLOAT)) + .put(Tuple.tuple("float", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + + .put(Tuple.tuple("double", ARRAY_FORMAT), new DoubleArrayIndexFieldData.Builder()) + .put(Tuple.tuple("double", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.DOUBLE)) + .put(Tuple.tuple("double", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + + .put(Tuple.tuple("byte", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.BYTE)) + .put(Tuple.tuple("byte", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BYTE)) + .put(Tuple.tuple("byte", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + + .put(Tuple.tuple("short", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.SHORT)) + .put(Tuple.tuple("short", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT)) + .put(Tuple.tuple("short", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + + .put(Tuple.tuple("int", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.INT)) + .put(Tuple.tuple("int", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT)) + .put(Tuple.tuple("int", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + + .put(Tuple.tuple("long", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.LONG)) + .put(Tuple.tuple("long", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG)) + .put(Tuple.tuple("long", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + + .put(Tuple.tuple("geo_point", ARRAY_FORMAT), new GeoPointDoubleArrayIndexFieldData.Builder()) + .put(Tuple.tuple("geo_point", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) .immutableMap(); } private final IndicesFieldDataCache indicesFieldDataCache; private final ConcurrentMap> loadedFieldData = ConcurrentCollections.newConcurrentMap(); + private final Map fieldDataCaches = Maps.newHashMap(); // no need for concurrency support, always used under lock IndexService indexService; + // public for testing public IndexFieldDataService(Index index) { this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, new IndicesFieldDataCache(ImmutableSettings.Builder.EMPTY_SETTINGS)); } @@ -114,6 +139,10 @@ public void clear() { fieldData.clear(); } loadedFieldData.clear(); + for (IndexFieldDataCache cache : fieldDataCaches.values()) { + cache.clear(); + } + fieldDataCaches.clear(); } } @@ -123,12 +152,29 @@ public void clearField(String fieldName) { if (fieldData != null) { fieldData.clear(); } + IndexFieldDataCache cache = fieldDataCaches.remove(fieldName); + if (cache != null) { + cache.clear(); + } } } public void clear(IndexReader reader) { - for (IndexFieldData indexFieldData : loadedFieldData.values()) { - indexFieldData.clear(reader); + synchronized (loadedFieldData) { + for (IndexFieldData indexFieldData : loadedFieldData.values()) { + indexFieldData.clear(reader); + } + for (IndexFieldDataCache cache : fieldDataCaches.values()) { + cache.clear(reader); + } + } + } + + public void onMappingUpdate() { + // synchronize to make sure to not miss field data instances that are being loaded + synchronized (loadedFieldData) { + // important: do not clear fieldDataCaches: the cache may be reused + loadedFieldData.clear(); } } @@ -163,18 +209,21 @@ public > IFD getForField(FieldMapper mapper) { throw new ElasticSearchIllegalArgumentException("failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type.getType()); } - IndexFieldDataCache cache; - // we default to node level cache, which in turn defaults to be unbounded - // this means changing the node level settings is simple, just set the bounds there - String cacheType = type.getSettings().get("cache", indexSettings.get("index.fielddata.cache", "node")); - if ("resident".equals(cacheType)) { - cache = new IndexFieldDataCache.Resident(indexService, fieldNames, type); - } else if ("soft".equals(cacheType)) { - cache = new IndexFieldDataCache.Soft(indexService, fieldNames, type); - } else if ("node".equals(cacheType)) { - cache = indicesFieldDataCache.buildIndexFieldDataCache(indexService, index, fieldNames, type); - } else { - throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]"); + IndexFieldDataCache cache = fieldDataCaches.get(fieldNames.indexName()); + if (cache == null) { + // we default to node level cache, which in turn defaults to be unbounded + // this means changing the node level settings is simple, just set the bounds there + String cacheType = type.getSettings().get("cache", indexSettings.get("index.fielddata.cache", "node")); + if ("resident".equals(cacheType)) { + cache = new IndexFieldDataCache.Resident(indexService, fieldNames, type); + } else if ("soft".equals(cacheType)) { + cache = new IndexFieldDataCache.Soft(indexService, fieldNames, type); + } else if ("node".equals(cacheType)) { + cache = indicesFieldDataCache.buildIndexFieldDataCache(indexService, index, fieldNames, type); + } else { + throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]"); + } + fieldDataCaches.put(fieldNames.indexName(), cache); } fieldData = builder.build(index, indexSettings, fieldNames, type, cache); @@ -184,4 +233,5 @@ public > IFD getForField(FieldMapper mapper) { } return (IFD) fieldData; } + } diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java new file mode 100644 index 0000000000000..b77fd7d3cd700 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java @@ -0,0 +1,68 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.AtomicReaderContext; +import org.elasticsearch.ElasticSearchIllegalStateException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.fieldcomparator.SortMode; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.FieldMapper.Names; +import org.elasticsearch.index.settings.IndexSettings; + +/** + * A field data implementation that forbids loading and will throw an {@link ElasticSearchIllegalStateException} if you try to load + * {@link AtomicFieldData} instances. + */ +public final class DisabledIndexFieldData extends AbstractIndexFieldData> { + + public static class Builder implements IndexFieldData.Builder { + @Override + public IndexFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) { + return new DisabledIndexFieldData(index, indexSettings, fieldNames, type, cache); + } + } + + public DisabledIndexFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) { + super(index, indexSettings, fieldNames, fieldDataType, cache); + } + + @Override + public boolean valuesOrdered() { + return false; + } + + @Override + public AtomicFieldData loadDirect(AtomicReaderContext context) throws Exception { + throw fail(); + } + + @Override + public IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, SortMode sortMode) { + throw fail(); + } + + private ElasticSearchIllegalStateException fail() { + return new ElasticSearchIllegalStateException("Field data loading is forbidden on " + getFieldNames().name()); + } + +} diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index f7bb47f35982e..7014cb947c891 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatService; import org.elasticsearch.index.codec.postingsformat.PostingsFormatService; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; @@ -76,6 +77,7 @@ public class MapperService extends AbstractIndexComponent implements Iterable typeListeners = new CopyOnWriteArrayList(); @Inject - public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService, + public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService, IndexFieldDataService fieldDataService, PostingsFormatService postingsFormatService, DocValuesFormatService docValuesFormatService, SimilarityLookupService similarityLookupService) { super(index, indexSettings); this.analysisService = analysisService; + this.fieldDataService = fieldDataService; this.documentParser = new DocumentMapperParser(index, indexSettings, analysisService, postingsFormatService, docValuesFormatService, similarityLookupService); this.searchAnalyzer = new SmartIndexNameSearchAnalyzer(analysisService.defaultSearchAnalyzer()); this.searchQuoteAnalyzer = new SmartIndexNameSearchQuoteAnalyzer(analysisService.defaultSearchQuoteAnalyzer()); @@ -278,6 +281,7 @@ private DocumentMapper merge(DocumentMapper mapper) { logger.debug("merging mapping for type [{}] resulted in conflicts: [{}]", mapper.type(), Arrays.toString(result.conflicts())); } } + fieldDataService.onMappingUpdate(); return oldMapper; } else { FieldMapperListener.Aggregator fieldMappersAgg = new FieldMapperListener.Aggregator(); diff --git a/src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java b/src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java new file mode 100644 index 0000000000000..b088035f11280 --- /dev/null +++ b/src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java @@ -0,0 +1,93 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.test.ElasticsearchIntegrationTest; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; + +public class DisabledFieldDataFormatTests extends ElasticsearchIntegrationTest { + + public void test() throws Exception { + createIndex("test"); + ensureGreen(); + for (int i = 0; i < 10; ++i) { + client().prepareIndex("test", "type", Integer.toString(i)).setSource("s", "value" + i).execute().actionGet(); + } + + refresh(); + + // disable field data + updateFormat("disabled"); + + SearchResponse resp = null; + // try to run something that relies on field data and make sure that it fails + try { + resp = client().prepareSearch("test").addAggregation(AggregationBuilders.terms("t").field("s")).execute().actionGet(); + assertTrue(resp.toString(), resp.getFailedShards() > 0); + } catch (SearchPhaseExecutionException e) { + // expected + } + + // enable it again + updateFormat("paged_bytes"); + + // try to run something that relies on field data and make sure that it works + resp = client().prepareSearch("test").addAggregation(AggregationBuilders.terms("t").field("s")).execute().actionGet(); + assertNoFailures(resp); + + // disable it again + updateFormat("disabled"); + + // this time, it should work because segments are already loaded + resp = client().prepareSearch("test").addAggregation(AggregationBuilders.terms("t").field("s")).execute().actionGet(); + assertNoFailures(resp); + + // but add more docs and the new segment won't be loaded + client().prepareIndex("test", "type", "-1").setSource("s", "value").execute().actionGet(); + refresh(); + try { + resp = client().prepareSearch("test").addAggregation(AggregationBuilders.terms("t").field("s")).execute().actionGet(); + assertTrue(resp.toString(), resp.getFailedShards() > 0); + } catch (SearchPhaseExecutionException e) { + // expected + } + } + + private void updateFormat(String format) throws Exception { + client().admin().indices().preparePutMapping("test").setType("type").setSource( + XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("s") + .field("type", "string") + .startObject("fielddata") + .field("format", format) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject()).execute().actionGet(); + } + +} diff --git a/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index e94ccd7166d7a..bdd4347fc0691 100644 --- a/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -19,6 +19,12 @@ package org.elasticsearch.index.fielddata; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.*; +import org.apache.lucene.store.RAMDirectory; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -26,10 +32,16 @@ import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.Mapper.BuilderContext; +import org.elasticsearch.index.mapper.MapperBuilders; import org.elasticsearch.index.mapper.core.*; import org.elasticsearch.test.ElasticsearchTestCase; import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; + +import static org.hamcrest.Matchers.instanceOf; public class IndexFieldDataServiceTests extends ElasticsearchTestCase { @@ -88,7 +100,7 @@ public void testGetForFieldDefaults() { public void testByPassDocValues() { final IndexFieldDataService ifdService = new IndexFieldDataService(new Index("test")); final BuilderContext ctx = new BuilderContext(null, new ContentPath(1)); - final StringFieldMapper stringMapper = new StringFieldMapper.Builder("string").tokenized(false).fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(ImmutableSettings.builder().put("format", "fst").build()).build(ctx); + final StringFieldMapper stringMapper = MapperBuilders.stringField("string").tokenized(false).fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(ImmutableSettings.builder().put("format", "fst").build()).build(ctx); ifdService.clear(); IndexFieldData fd = ifdService.getForField(stringMapper); assertTrue(fd instanceof FSTBytesIndexFieldData); @@ -105,15 +117,53 @@ public void testByPassDocValues() { assertTrue(fd instanceof PackedArrayIndexFieldData); } - final FloatFieldMapper floatMapper = new FloatFieldMapper.Builder("float").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx); + final FloatFieldMapper floatMapper = MapperBuilders.floatField("float").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx); ifdService.clear(); fd = ifdService.getForField(floatMapper); assertTrue(fd instanceof FloatArrayIndexFieldData); - final DoubleFieldMapper doubleMapper = new DoubleFieldMapper.Builder("double").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx); + final DoubleFieldMapper doubleMapper = MapperBuilders.doubleField("double").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx); ifdService.clear(); fd = ifdService.getForField(doubleMapper); assertTrue(fd instanceof DoubleArrayIndexFieldData); } + public void testChangeFieldDataFormat() throws Exception { + final IndexFieldDataService ifdService = new IndexFieldDataService(new Index("test")); + final BuilderContext ctx = new BuilderContext(null, new ContentPath(1)); + final StringFieldMapper mapper1 = MapperBuilders.stringField("s").tokenized(false).fieldDataSettings(ImmutableSettings.builder().put(FieldDataType.FORMAT_KEY, "paged_bytes").build()).build(ctx); + final IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())); + Document doc = new Document(); + doc.add(new StringField("s", "thisisastring", Store.NO)); + writer.addDocument(doc); + final IndexReader reader1 = DirectoryReader.open(writer, true); + IndexFieldData ifd = ifdService.getForField(mapper1); + assertThat(ifd, instanceOf(PagedBytesIndexFieldData.class)); + Set oldSegments = Collections.newSetFromMap(new IdentityHashMap()); + for (AtomicReaderContext arc : reader1.leaves()) { + oldSegments.add(arc.reader()); + AtomicFieldData afd = ifd.load(arc); + assertThat(afd, instanceOf(PagedBytesAtomicFieldData.class)); + } + // write new segment + writer.addDocument(doc); + final IndexReader reader2 = DirectoryReader.open(writer, true); + final StringFieldMapper mapper2 = MapperBuilders.stringField("s").tokenized(false).fieldDataSettings(ImmutableSettings.builder().put(FieldDataType.FORMAT_KEY, "fst").build()).build(ctx); + ifdService.onMappingUpdate(); + ifd = ifdService.getForField(mapper2); + assertThat(ifd, instanceOf(FSTBytesIndexFieldData.class)); + for (AtomicReaderContext arc : reader2.leaves()) { + AtomicFieldData afd = ifd.load(arc); + if (oldSegments.contains(arc.reader())) { + assertThat(afd, instanceOf(PagedBytesAtomicFieldData.class)); + } else { + assertThat(afd, instanceOf(FSTBytesAtomicFieldData.class)); + } + } + reader1.close(); + reader2.close(); + writer.close(); + writer.getDirectory().close(); + } + } diff --git a/src/test/java/org/elasticsearch/index/mapper/MapperTestUtils.java b/src/test/java/org/elasticsearch/index/mapper/MapperTestUtils.java index a385c3607d481..786c00f2a2866 100644 --- a/src/test/java/org/elasticsearch/index/mapper/MapperTestUtils.java +++ b/src/test/java/org/elasticsearch/index/mapper/MapperTestUtils.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatService; import org.elasticsearch.index.codec.postingsformat.PostingsFormatService; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.settings.IndexSettingsModule; import org.elasticsearch.index.similarity.SimilarityLookupService; import org.elasticsearch.indices.analysis.IndicesAnalysisModule; @@ -53,8 +54,12 @@ public static DocumentMapperParser newParser(Settings indexSettings) { } public static MapperService newMapperService() { - return new MapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, new Environment(), newAnalysisService(), - new PostingsFormatService(new Index("test")), new DocValuesFormatService(new Index("test")), newSimilarityLookupService()); + return newMapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS); + } + + public static MapperService newMapperService(Index index, Settings indexSettings) { + return new MapperService(index, indexSettings, new Environment(), newAnalysisService(), new IndexFieldDataService(index), + new PostingsFormatService(index), new DocValuesFormatService(index), newSimilarityLookupService()); } public static AnalysisService newAnalysisService() { diff --git a/src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java b/src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java index d310199241116..5639045a94968 100644 --- a/src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java +++ b/src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java @@ -36,15 +36,14 @@ import org.elasticsearch.common.lucene.search.XFilteredQuery; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; -import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache; import org.elasticsearch.index.cache.id.IdCache; import org.elasticsearch.index.cache.id.SimpleIdCacheTests; import org.elasticsearch.index.cache.id.simple.SimpleIdCache; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperTestUtils; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; @@ -328,9 +327,7 @@ static SearchContext createSearchContext(String indexName, String parentType, St final IdCache idCache = new SimpleIdCache(index, ImmutableSettings.EMPTY); final CacheRecycler cacheRecycler = new CacheRecycler(ImmutableSettings.EMPTY); Settings settings = ImmutableSettings.EMPTY; - MapperService mapperService = new MapperService( - index, settings, new Environment(), new AnalysisService(index), null, null, null - ); + MapperService mapperService = MapperTestUtils.newMapperService(index, settings); mapperService.merge( childType, new CompressedString(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), true );