From a8d6c64b3aaa4ef7e52ccf841bad98d2ae71eeb6 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 23 Oct 2025 14:25:39 +0100 Subject: [PATCH 01/11] Default indexed to false for keyword dimension fields --- .../index/mapper/KeywordFieldMapper.java | 13 ++++++------- .../index/mapper/KeywordFieldMapperTests.java | 14 ++------------ 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 18d748713fa64..e7ba1e46f1b61 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -167,7 +167,7 @@ private static KeywordFieldMapper toType(FieldMapper in) { public static final class Builder extends FieldMapper.DimensionBuilder { - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType.stored(), false); @@ -279,21 +279,20 @@ private Builder( null ).acceptsNull(); this.script.precludesParameters(nullValue); - addScriptValidation(script, indexed, hasDocValues); this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { + if (v && (hasDocValues.getValue() == false)) { throw new IllegalArgumentException( "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] requires that [" - + indexed.name - + "] and [" + hasDocValues.name - + "] are true" + + "] is true" ); } }).precludesParameters(normalizer); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> this.dimension.get() == false); + addScriptValidation(script, indexed, hasDocValues); this.ignoreAboveDefault = ignoreAboveDefault; this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault); this.indexSortConfig = indexSortConfig; @@ -457,7 +456,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType public KeywordFieldMapper build(MapperBuilderContext context) { FieldType fieldtype = resolveFieldType( enableDocValuesSkipper, - forceDocValuesSkipper, + forceDocValuesSkipper || (this.dimension.get() && this.indexed.get() == false), hasDocValues, indexCreatedVersion, indexSortConfig, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 845b2c498e252..ae779c971db22 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -457,7 +457,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } { @@ -467,17 +467,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } } From 1293cf4120cfd88e7a087dad46a41c7f6a2c8d78 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 23 Oct 2025 13:37:40 +0000 Subject: [PATCH 02/11] [CI] Auto commit changes from spotless --- .../elasticsearch/index/mapper/KeywordFieldMapper.java | 6 +----- .../index/mapper/KeywordFieldMapperTests.java | 10 ++-------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e7ba1e46f1b61..bd6890701db5e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -283,11 +283,7 @@ private Builder( this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> { if (v && (hasDocValues.getValue() == false)) { throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + hasDocValues.name - + "] is true" + "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] requires that [" + hasDocValues.name + "] is true" ); } }).precludesParameters(normalizer); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index ae779c971db22..541d6a08cb693 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -455,20 +455,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } From 40bfeb3d30082a0bebcdae9b094c185be2b909ae Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 23 Oct 2025 15:26:04 +0100 Subject: [PATCH 03/11] Same for IpFieldMapper --- .../index/mapper/IpFieldMapper.java | 20 ++++++++++++------- .../index/mapper/IpFieldMapperTests.java | 14 ++----------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 644ab73a0217b..e9c99ef24be09 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -73,7 +73,7 @@ private static IpFieldMapper toType(FieldMapper in) { public static final class Builder extends FieldMapper.DimensionBuilder { - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); @@ -108,21 +108,20 @@ public Builder( this.indexCreatedVersion = indexCreatedVersion; this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault); this.script.precludesParameters(nullValue, ignoreMalformed); - addScriptValidation(script, indexed, hasDocValues); this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { + if (v && (hasDocValues.getValue() == false)) { throw new IllegalArgumentException( "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] requires that [" - + indexed.name - + "] and [" + hasDocValues.name - + "] are true" + + "] is true" ); } }); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> this.dimension.get() == false); this.indexSourceKeepMode = indexSourceKeepMode; + addScriptValidation(script, indexed, hasDocValues); } Builder nullValue(String nullValue) { @@ -190,6 +189,9 @@ private IndexType indexType() { if (indexCreatedVersion.isLegacyIndexVersion()) { return hasDocValues.get() ? IndexType.archivedPoints() : IndexType.NONE; } + if (dimension.get()) { + return IndexType.skippers(); + } return IndexType.points(indexed.get(), hasDocValues.get()); } @@ -659,7 +661,11 @@ private void indexValue(DocumentParserContext context, ESInetAddressPoint addres doc.add(address); } if (hasDocValues) { - doc.add(new SortedSetDocValuesField(fieldType().name(), address.binaryValue())); + if (indexed == false) { + doc.add(SortedSetDocValuesField.indexedField(fieldType().name(), address.binaryValue())); + } else { + doc.add(new SortedSetDocValuesField(fieldType().name(), address.binaryValue())); + } } else if (stored || indexed) { context.addToFieldNames(fieldType().name()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 2ca2a7f8eb62d..5cd3b5953d6a4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -234,7 +234,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } { @@ -244,17 +244,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } } From 54f19a0e4713daec4a2448f61f786846b253db4b Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 23 Oct 2025 14:33:42 +0000 Subject: [PATCH 04/11] [CI] Auto commit changes from spotless --- .../org/elasticsearch/index/mapper/IpFieldMapper.java | 6 +----- .../elasticsearch/index/mapper/IpFieldMapperTests.java | 10 ++-------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index e9c99ef24be09..07f8dbe6fba99 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -111,11 +111,7 @@ public Builder( this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { if (v && (hasDocValues.getValue() == false)) { throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + hasDocValues.name - + "] is true" + "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] requires that [" + hasDocValues.name + "] is true" ); } }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 5cd3b5953d6a4..642396f8558fb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -232,20 +232,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } From 12cf47905d5b8833e74cafa015cfb1d2b7d8d7ee Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 11 Nov 2025 13:45:11 +0000 Subject: [PATCH 05/11] wip --- .../elasticsearch/index/IndexVersions.java | 1 + .../index/mapper/FieldMapper.java | 15 +++ .../index/mapper/GeoPointFieldMapper.java | 2 +- .../index/mapper/IpFieldMapper.java | 15 +-- .../index/mapper/KeywordFieldMapper.java | 11 +- .../index/mapper/NumberFieldMapper.java | 105 ++++++++++-------- .../index/mapper/TimeSeriesParams.java | 12 +- 7 files changed, 92 insertions(+), 69 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 172bdc67e7872..053fbd010af12 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -193,6 +193,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1); public static final IndexVersion SKIPPERS_ENABLED_BY_DEFAULT = def(9_043_0_00, Version.LUCENE_10_3_1); public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID = def(9_044_0_00, Version.LUCENE_10_3_1); + public static final IndexVersion TIME_SERIES_DIMENSIONS_USE_SKIPPERS = def(9_045_0_00, Version.LUCENE_10_3_1); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index dd35576299a0e..412c7648f2acf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1314,6 +1314,21 @@ public static Parameter indexParam(Function initi return Parameter.boolParam("index", false, initializer, defaultValue); } + public static Parameter indexParam( + Function initializer, + IndexSettings indexSettings, + Supplier isDimension + ) { + return Parameter.boolParam( + "index", + false, + initializer, + () -> isDimension.get() == false + || indexSettings.useDocValuesSkipper() == false + || indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS) + ); + } + public static Parameter storeParam(Function initializer, boolean defaultValue) { return Parameter.boolParam("store", false, initializer, defaultValue); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 655c25924d887..56ec1e796f352 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -143,7 +143,7 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe } }); // We allow `time_series_dimension` parameter to be parsed, but only allow it to be `false` - this.dimension = TimeSeriesParams.dimensionParam(m -> false).addValidator(v -> { + this.dimension = TimeSeriesParams.dimensionParam(m -> false, () -> true).addValidator(v -> { if (v) { throw new IllegalArgumentException( "Parameter [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] cannot be set to geo_point" diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index a072b9f48e903..0d40b7ccca297 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -105,18 +105,9 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe IGNORE_MALFORMED_SETTING.get(indexSettings.getSettings()) ); this.script.precludesParameters(nullValue, ignoreMalformed); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (hasDocValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + hasDocValues.name - + "] is true" - ); - } - }); - this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> this.dimension.get() == false); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, + () -> this.dimension.get() == false && indexSettings.); addScriptValidation(script, indexed, hasDocValues); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 765c2124fd531..849e1cbccb0e7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -256,14 +256,9 @@ public Builder( this.script.precludesParameters(nullValue); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> { - if (v && (hasDocValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] requires that [" + hasDocValues.name + "] is true" - ); - } - }).precludesParameters(normalizer); - this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> this.dimension.get() == false); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension(), hasDocValues::get) + .precludesParameters(normalizer); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, indexSettings, dimension); addScriptValidation(script, indexed, hasDocValues); this.ignoreAbove = Parameter.ignoreAboveParam( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 434e74fefb90a..de87a4d1cca75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -174,19 +174,7 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, IndexSetti return true; } }); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + indexed.name - + "] and [" - + hasDocValues.name - + "] are true" - ); - } - }); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE, MetricType.COUNTER).addValidator(v -> { if (v != null && hasDocValues.getValue() == false) { @@ -224,6 +212,13 @@ private IndexType indexType() { if (indexSettings.getIndexVersionCreated().isLegacyIndexVersion()) { return IndexType.archivedPoints(); } + if (indexSettings.useDocValuesSkipper() + && indexed.get() == false + && hasDocValues.get() + && dimension.get() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) { + return IndexType.skippers(); + } return IndexType.points(indexed.get(), hasDocValues.get()); } @@ -413,13 +408,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final float f = value.floatValue(); - if (indexed) { + if (indexType.hasPoints()) { document.add(new HalfFloatPoint(name, f)); } - if (docValued) { - document.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(f))); + if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, HalfFloatPoint.halfFloatToSortableShort(f))); + } else { + document.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(f))); + } } if (stored) { document.add(new StoredField(name, f)); @@ -606,13 +605,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final float f = value.floatValue(); - if (indexed && docValued) { + if (indexType.hasPoints() && indexType.hasDocValues()) { document.add(new FloatField(name, f, Field.Store.NO)); - } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f))); - } else if (indexed) { + } else if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.floatToSortableInt(f))); + } else { + document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f))); + } + } else if (indexType.hasPoints()) { document.add(new FloatPoint(name, f)); } if (stored) { @@ -766,13 +769,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final double d = value.doubleValue(); - if (indexed && docValued) { + if (indexType.hasPoints() && indexType.hasDocValues()) { document.add(new DoubleField(name, d, Field.Store.NO)); - } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d))); - } else if (indexed) { + } else if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.doubleToSortableLong(d))); + } else { + document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d))); + } + } else if (indexType.hasPoints()) { document.add(new DoublePoint(name, d)); } if (stored) { @@ -905,8 +912,8 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { - INTEGER.addFields(document, name, value, indexed, docValued, stored); + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { + INTEGER.addFields(document, name, value, indexType, stored); } @Override @@ -1033,8 +1040,8 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { - INTEGER.addFields(document, name, value, indexed, docValued, stored); + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { + INTEGER.addFields(document, name, value, indexType, stored); } @Override @@ -1230,13 +1237,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final int i = value.intValue(); - if (indexed && docValued) { + if (indexType.hasPoints() && indexType.hasDocValues()) { document.add(new IntField(name, i, Field.Store.NO)); - } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, i)); - } else if (indexed) { + } else if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, i)); + } else { + document.add(new SortedNumericDocValuesField(name, i)); + } + } else if (indexType.hasPoints()) { document.add(new IntPoint(name, i)); } if (stored) { @@ -1392,13 +1403,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final long l = value.longValue(); - if (indexed && docValued) { + if (indexType.hasPoints() && indexType.hasDocValues()) { document.add(new LongField(name, l, Field.Store.NO)); - } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, l)); - } else if (indexed) { + } else if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, l)); + } else { + document.add(new SortedNumericDocValuesField(name, l)); + } + } else if (indexType.hasPoints()) { document.add(new LongPoint(name, l)); } if (stored) { @@ -1545,16 +1560,14 @@ public abstract Query rangeQuery( * @param document document to add fields to * @param name field name * @param value value to map - * @param indexed whether or not the field is indexed - * @param docValued whether or not doc values should be added + * @param indexType an IndexType describing the index structures to be added * @param stored whether or not the field is stored */ public abstract void addFields( LuceneDocument document, String name, Number value, - boolean indexed, - boolean docValued, + IndexType indexType, boolean stored ); @@ -2268,7 +2281,7 @@ public void indexValue(DocumentParserContext context, Number numericValue) { if (dimension && numericValue != null) { context.getRoutingFields().addLong(fieldType().name(), numericValue.longValue()); } - fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored); + fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, fieldType().indexType, stored); if (false == allowMultipleValues && (indexed || hasDocValues || stored)) { // the last field is the current field, Add to the key map, so that we can validate if it has been added diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java index 1f4676d60814f..1db2221be088c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.Locale; +import java.util.function.BooleanSupplier; import java.util.function.Function; /** @@ -88,8 +89,15 @@ public static FieldMapper.Parameter metricParam(Function dimensionParam(Function initializer) { - return FieldMapper.Parameter.boolParam(TIME_SERIES_DIMENSION_PARAM, false, initializer, false); + public static FieldMapper.Parameter dimensionParam(Function initializer, BooleanSupplier hasDocValues) { + return FieldMapper.Parameter.boolParam(TIME_SERIES_DIMENSION_PARAM, false, initializer, false) + .addValidator(v -> { + if (v && (hasDocValues.getAsBoolean() == false)) { + throw new IllegalArgumentException( + "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] requires that [doc_values] is true" + ); + } + }); } } From eb8681652ee2a3f2424031cfe3e00758f260c4d2 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 11 Nov 2025 14:19:58 +0000 Subject: [PATCH 06/11] Use IndexType in TermBasedFieldType constructors In preparation for doc-value skippers being enabled in some keyword field types, TermBasedFieldType should take IndexType directly in its constructor rather than booleans for indexed and docvalues. --- .../extras/MatchOnlyTextFieldMapper.java | 3 +- .../extras/SearchAsYouTypeFieldMapper.java | 8 ++--- .../join/mapper/ParentIdFieldMapper.java | 3 +- .../join/mapper/ParentJoinFieldMapper.java | 3 +- .../icu/ICUCollationKeywordFieldMapper.java | 13 ++++---- .../index/mapper/BooleanFieldMapper.java | 2 +- .../index/mapper/CompletionFieldMapper.java | 8 ++++- .../index/mapper/FieldNamesFieldMapper.java | 2 +- .../index/mapper/IdFieldMapper.java | 2 +- .../index/mapper/IgnoredFieldMapper.java | 4 +-- .../mapper/IgnoredSourceFieldMapper.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 14 ++++----- .../index/mapper/LegacyTypeFieldMapper.java | 2 +- .../index/mapper/NestedPathFieldMapper.java | 2 +- .../index/mapper/RoutingFieldMapper.java | 2 +- .../index/mapper/StringFieldType.java | 11 ++----- .../index/mapper/TermBasedFieldType.java | 11 ++----- .../index/mapper/TextFamilyFieldType.java | 5 ++- .../index/mapper/TextFieldMapper.java | 13 +++++--- .../flattened/FlattenedFieldMapper.java | 30 +++++------------- .../index/mapper/MappingLookupTests.java | 2 +- .../KeyedFlattenedFieldTypeTests.java | 6 ++-- .../RootFlattenedFieldTypeTests.java | 31 ++++++++++++++++--- .../index/mapper/MockFieldMapper.java | 2 +- .../patterntext/PatternTextFieldType.java | 3 +- .../CountedKeywordFieldMapper.java | 9 +++--- .../VersionStringFieldMapper.java | 9 +++++- 27 files changed, 107 insertions(+), 95 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index d54764d803101..16995f00f010b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -189,7 +190,7 @@ public MatchOnlyTextFieldType( boolean storedFieldInBinaryFormat, KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate ) { - super(name, true, false, false, tsi, meta, isSyntheticSource, withinMultiField); + super(name, IndexType.terms(true, false), false, tsi, meta, isSyntheticSource, withinMultiField); this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); this.textFieldType = new TextFieldType(name, isSyntheticSource, withinMultiField, syntheticSourceDelegate); this.storedFieldInBinaryFormat = storedFieldInBinaryFormat; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java index ba4fb0b644258..a82609a42746b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperParsingException; @@ -306,9 +307,8 @@ static class SearchAsYouTypeFieldType extends StringFieldType { ) { super( name, - fieldType.indexOptions() != IndexOptions.NONE, + IndexType.terms(fieldType.indexOptions() != IndexOptions.NONE, false), fieldType.stored(), - false, new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer), meta ); @@ -433,7 +433,7 @@ static final class PrefixFieldType extends StringFieldType { final String parentField; PrefixFieldType(String parentField, TextSearchInfo textSearchInfo, int minChars, int maxChars) { - super(parentField + PREFIX_FIELD_SUFFIX, true, false, false, textSearchInfo, Collections.emptyMap()); + super(parentField + PREFIX_FIELD_SUFFIX, IndexType.terms(true, false), false, textSearchInfo, Collections.emptyMap()); this.minChars = minChars; this.maxChars = maxChars; this.parentField = parentField; @@ -574,7 +574,7 @@ static class ShingleFieldType extends StringFieldType { PrefixFieldType prefixFieldType; ShingleFieldType(String name, int shingleSize, TextSearchInfo textSearchInfo) { - super(name, true, false, false, textSearchInfo, Collections.emptyMap()); + super(name, IndexType.terms(true, false), false, textSearchInfo, Collections.emptyMap()); this.shingleSize = shingleSize; } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 46cec9a0b7a5f..00d8dfa9b6d3d 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -44,7 +45,7 @@ public static final class ParentIdFieldType extends StringFieldType { private final boolean eagerGlobalOrdinals; public ParentIdFieldType(String name, boolean eagerGlobalOrdinals) { - super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(name, IndexType.terms(true, true), false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); this.eagerGlobalOrdinals = eagerGlobalOrdinals; } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index fd503bf75178c..a016f718baeea 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -151,7 +152,7 @@ public static final class JoinFieldType extends StringFieldType { private final Joiner joiner; private JoinFieldType(String name, Joiner joiner, Map meta) { - super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + super(name, IndexType.terms(true, true), false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.joiner = joiner; } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 9dd959981701c..292c51d5d3ab6 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.SourceValueFetcher; @@ -62,26 +63,25 @@ public static final class CollationFieldType extends StringFieldType { public CollationFieldType( String name, - boolean isSearchable, + IndexType indexType, boolean isStored, - boolean hasDocValues, Collator collator, String nullValue, int ignoreAbove, Map meta ) { - super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + super(name, indexType, isStored, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.collator = collator; this.nullValue = nullValue; this.ignoreAbove = ignoreAbove; } public CollationFieldType(String name, boolean searchable, Collator collator) { - this(name, searchable, false, true, collator, null, Integer.MAX_VALUE, Collections.emptyMap()); + this(name, IndexType.terms(searchable, true), false, collator, null, Integer.MAX_VALUE, Collections.emptyMap()); } public CollationFieldType(String name, Collator collator) { - this(name, true, false, true, collator, null, Integer.MAX_VALUE, Collections.emptyMap()); + this(name, IndexType.terms(true, true), false, collator, null, Integer.MAX_VALUE, Collections.emptyMap()); } @Override @@ -327,9 +327,8 @@ public ICUCollationKeywordFieldMapper build(MapperBuilderContext context) { final Collator collator = params.buildCollator(); CollationFieldType ft = new CollationFieldType( context.buildFullName(leafName()), - indexed.getValue(), + IndexType.terms(indexed.get(), hasDocValues.get()), stored.getValue(), - hasDocValues.getValue(), collator, nullValue.getValue(), ignoreAbove.getValue(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 75bd68344daed..d59c6c31deafa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -224,7 +224,7 @@ public BooleanFieldType( boolean isDimension, boolean isSyntheticSource ) { - super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + super(name, IndexType.terms(isIndexed, hasDocValues), isStored, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.nullValue = nullValue; this.scriptValues = scriptValues; this.isDimension = isDimension; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index adcb946d7afc4..9af9396438674 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -251,7 +251,13 @@ public static final class CompletionFieldType extends TermBasedFieldType { private ContextMappings contextMappings = null; public CompletionFieldType(String name, NamedAnalyzer searchAnalyzer, Map meta) { - super(name, true, false, false, new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchAnalyzer), meta); + super( + name, + IndexType.terms(true, false), + false, + new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchAnalyzer), + meta + ); } public void setContextMappings(ContextMappings contextMappings) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index 425e3c664c262..7bfbab8b09ded 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -120,7 +120,7 @@ public static FieldNamesFieldType get(boolean enabled) { } private FieldNamesFieldType(boolean enabled) { - super(Defaults.NAME, true, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(Defaults.NAME, IndexType.terms(true, false), false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); this.enabled = enabled; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 61f5fa9abcf0a..adabb686b8686 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -88,7 +88,7 @@ public static Field syntheticIdField(String id) { protected abstract static class AbstractIdFieldType extends TermBasedFieldType { public AbstractIdFieldType() { - super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(NAME, IndexType.terms(true, false), true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index 70b43d202b3d6..acb31af1ff44f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -57,7 +57,7 @@ private static MetadataFieldMapper getInstance(IndexVersion indexVersion) { public static final class LegacyIgnoredFieldType extends StringFieldType { private LegacyIgnoredFieldType() { - super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(NAME, IndexType.terms(true, false), true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @Override @@ -88,7 +88,7 @@ public Query existsQuery(SearchExecutionContext context) { public static final class IgnoredFieldType extends StringFieldType { private IgnoredFieldType() { - super(NAME, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(NAME, IndexType.terms(true, true), false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 742f9751ce280..cd3ff11b8f214 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -142,7 +142,7 @@ static final class IgnoredValuesFieldMapperType extends StringFieldType { private static final IgnoredValuesFieldMapperType INSTANCE = new IgnoredValuesFieldMapperType(); private IgnoredValuesFieldMapperType() { - super(NAME, false, true, false, TextSearchInfo.NONE, Collections.emptyMap()); + super(NAME, IndexType.NONE, true, TextSearchInfo.NONE, Collections.emptyMap()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 328988e235fcb..685e272604e1f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -513,9 +513,11 @@ public KeywordFieldType( ) { super( name, - fieldType.indexOptions() != IndexOptions.NONE && builder.indexCreatedVersion.isLegacyIndexVersion() == false, + IndexType.terms( + fieldType.indexOptions() != IndexOptions.NONE && builder.indexCreatedVersion.isLegacyIndexVersion() == false, + builder.hasDocValues.get() + ), fieldType.stored(), - builder.hasDocValues.getValue(), textSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, quoteAnalyzer), builder.meta.getValue(), isSyntheticSource, @@ -539,7 +541,7 @@ public KeywordFieldType(String name) { } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { - super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta, false, false); + super(name, IndexType.terms(isIndexed, hasDocValues), false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta, false, false); this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; @@ -552,8 +554,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma public KeywordFieldType(String name, FieldType fieldType) { super( name, - fieldType.indexOptions() != IndexOptions.NONE, - false, + IndexType.terms(fieldType.indexOptions() != IndexOptions.NONE, false), false, textSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap(), @@ -572,9 +573,8 @@ public KeywordFieldType(String name, FieldType fieldType) { public KeywordFieldType(String name, NamedAnalyzer analyzer) { super( name, - true, + IndexType.terms(true, true), false, - true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap(), false, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java index c6f1b490a2be2..4498655b9b8a8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -48,7 +48,7 @@ public Map indexAnalyzers() { private static final class LegacyTypeFieldType extends TermBasedFieldType { LegacyTypeFieldType() { - super(NAME, false, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(NAME, IndexType.docValuesOnly(), true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java index 1cd752dc34403..690e442c97807 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java @@ -52,7 +52,7 @@ public static Field field(IndexVersion version, String path) { public static final class NestedPathFieldType extends StringFieldType { private NestedPathFieldType(String name) { - super(name, true, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(name, IndexType.terms(true, false), false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index c5c1a2f9bc9d5..0b44f2724fb5c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -62,7 +62,7 @@ public RoutingFieldMapper build() { static final class RoutingFieldType extends StringFieldType { private RoutingFieldType() { - super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(NAME, IndexType.terms(true, false), true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java index ceb96b87a0983..7d7b7b4b0b010 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java @@ -43,15 +43,8 @@ public abstract class StringFieldType extends TermBasedFieldType { private static final Pattern WILDCARD_PATTERN = Pattern.compile("(\\\\.)|([?*]+)"); - public StringFieldType( - String name, - boolean isIndexed, - boolean isStored, - boolean hasDocValues, - TextSearchInfo textSearchInfo, - Map meta - ) { - super(name, isIndexed, isStored, hasDocValues, textSearchInfo, meta); + public StringFieldType(String name, IndexType indexType, boolean isStored, TextSearchInfo textSearchInfo, Map meta) { + super(name, indexType, isStored, textSearchInfo, meta); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java index 96f1c3fe281d9..3399df9a8ae5b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java @@ -28,15 +28,8 @@ public abstract class TermBasedFieldType extends SimpleMappedFieldType { protected final TextSearchInfo textSearchInfo; - public TermBasedFieldType( - String name, - boolean isIndexed, - boolean isStored, - boolean hasDocValues, - TextSearchInfo textSearchInfo, - Map meta - ) { - super(name, IndexType.terms(isIndexed, hasDocValues), isStored, meta); + public TermBasedFieldType(String name, IndexType indexType, boolean isStored, TextSearchInfo textSearchInfo, Map meta) { + super(name, indexType, isStored, meta); this.textSearchInfo = textSearchInfo; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java index de03b116f0c19..762d6ff28aaef 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java @@ -26,15 +26,14 @@ public abstract class TextFamilyFieldType extends StringFieldType { public TextFamilyFieldType( String name, - boolean isIndexed, + IndexType indexType, boolean isStored, - boolean hasDocValues, TextSearchInfo textSearchInfo, Map meta, boolean isSyntheticSourceEnabled, boolean isWithinMultiField ) { - super(name, isIndexed, isStored, hasDocValues, textSearchInfo, meta); + super(name, indexType, isStored, textSearchInfo, meta); this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; this.isWithinMultiField = isWithinMultiField; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index a0652fc669d63..57ab9d9a9b2f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -582,7 +582,13 @@ private static final class PrefixFieldType extends StringFieldType { final TextFieldType parentField; PrefixFieldType(TextFieldType parentField, int minChars, int maxChars) { - super(parentField.name() + FAST_PREFIX_SUFFIX, true, false, false, parentField.getTextSearchInfo(), Collections.emptyMap()); + super( + parentField.name() + FAST_PREFIX_SUFFIX, + IndexType.terms(true, false), + false, + parentField.getTextSearchInfo(), + Collections.emptyMap() + ); this.minChars = minChars; this.maxChars = maxChars; this.parentField = parentField; @@ -707,7 +713,7 @@ public TextFieldType( boolean eagerGlobalOrdinals, boolean indexPhrases ) { - super(name, indexed, stored, false, tsi, meta, isSyntheticSource, isWithinMultiField); + super(name, indexed ? IndexType.terms(true, false) : IndexType.NONE, stored, tsi, meta, isSyntheticSource, isWithinMultiField); fielddata = false; // TODO block loader could use a "fast loading" delegate which isn't always the same - but frequently is. this.syntheticSourceDelegate = Optional.ofNullable(syntheticSourceDelegate); @@ -718,9 +724,8 @@ public TextFieldType( public TextFieldType(String name, boolean indexed, boolean stored, Map meta) { super( name, - indexed, + IndexType.terms(indexed, false), stored, - false, new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), meta, false, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 2a9e52231979d..547f0def112f6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -243,8 +243,7 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { } MappedFieldType ft = new RootFlattenedFieldType( context.buildFullName(leafName()), - indexed.get(), - hasDocValues.get(), + IndexType.terms(indexed.get(), hasDocValues.get()), meta.get(), splitQueriesOnWhitespace.get(), eagerGlobalOrdinals.get(), @@ -273,8 +272,7 @@ public boolean isDimension() { KeyedFlattenedFieldType( String rootName, - boolean indexed, - boolean hasDocValues, + IndexType indexType, String key, boolean splitQueriesOnWhitespace, Map meta, @@ -282,9 +280,8 @@ public boolean isDimension() { ) { super( rootName + KEYED_FIELD_SUFFIX, - indexed, + indexType, false, - hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta ); @@ -294,15 +291,7 @@ public boolean isDimension() { } private KeyedFlattenedFieldType(String rootName, String key, RootFlattenedFieldType ref) { - this( - rootName, - ref.indexType().hasTerms(), - ref.hasDocValues(), - key, - ref.splitQueriesOnWhitespace, - ref.meta(), - ref.dimensions.contains(key) - ); + this(rootName, ref.indexType(), key, ref.splitQueriesOnWhitespace, ref.meta(), ref.dimensions.contains(key)); } @Override @@ -685,20 +674,18 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme RootFlattenedFieldType( String name, - boolean indexed, - boolean hasDocValues, + IndexType indexType, Map meta, boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, IgnoreAbove ignoreAbove ) { - this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove); + this(name, indexType, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove); } RootFlattenedFieldType( String name, - boolean indexed, - boolean hasDocValues, + IndexType indexType, Map meta, boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, @@ -707,9 +694,8 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme ) { super( name, - indexed, + indexType, false, - hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index aab50300270b7..6778ffc8e7c9b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -239,7 +239,7 @@ public boolean incrementToken() { static class FakeFieldType extends TermBasedFieldType { private FakeFieldType(String name) { - super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(name, IndexType.terms(true, true), false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index dadeae3cc654c..6e1d8143bb24d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; @@ -42,7 +43,7 @@ public class KeyedFlattenedFieldTypeTests extends FieldTypeTestCase { private static KeyedFlattenedFieldType createFieldType() { - return new KeyedFlattenedFieldType("field", true, true, "key", false, Collections.emptyMap(), false); + return new KeyedFlattenedFieldType("field", IndexType.terms(true, true), "key", false, Collections.emptyMap(), false); } public void testIndexedValueForSearch() { @@ -69,8 +70,7 @@ public void testTermQuery() { KeyedFlattenedFieldType unsearchable = new KeyedFlattenedFieldType( "field", - false, - true, + IndexType.terms(false, true), "key", false, Collections.emptyMap(), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index 3127b2c60d0f5..eda9a793e73c3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; @@ -38,7 +39,14 @@ public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { private static final Mapper.IgnoreAbove IGNORE_ABOVE = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { - return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, new Mapper.IgnoreAbove(ignoreAbove)); + return new RootFlattenedFieldType( + "field", + IndexType.terms(true, true), + Collections.emptyMap(), + false, + false, + new Mapper.IgnoreAbove(ignoreAbove) + ); } public void testValueForDisplay() { @@ -60,8 +68,7 @@ public void testTermQuery() { RootFlattenedFieldType unsearchable = new RootFlattenedFieldType( "field", - false, - true, + IndexType.terms(false, true), Collections.emptyMap(), false, false, @@ -72,10 +79,24 @@ public void testTermQuery() { } public void testExistsQuery() { - RootFlattenedFieldType ft = new RootFlattenedFieldType("field", true, false, Collections.emptyMap(), false, false, IGNORE_ABOVE); + RootFlattenedFieldType ft = new RootFlattenedFieldType( + "field", + IndexType.terms(true, false), + Collections.emptyMap(), + false, + false, + IGNORE_ABOVE + ); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); - RootFlattenedFieldType withDv = new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, IGNORE_ABOVE); + RootFlattenedFieldType withDv = new RootFlattenedFieldType( + "field", + IndexType.terms(true, true), + Collections.emptyMap(), + false, + false, + IGNORE_ABOVE + ); assertEquals(new FieldExistsQuery("field"), withDv.existsQuery(null)); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 755903e3ea7ae..3e3bb30e2ed0f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -45,7 +45,7 @@ static String findSimpleName(String fullName) { public static class FakeFieldType extends TermBasedFieldType { public FakeFieldType(String name) { - super(name, true, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + super(name, IndexType.terms(true, false), false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @Override diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index fd072cd8157e9..c6662ffda77fa 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BlockStoredFieldsReader; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -77,7 +78,7 @@ public class PatternTextFieldType extends TextFamilyFieldType { ) { // Though this type is based on doc_values, hasDocValues is set to false as the pattern_text type is not aggregatable. // This does not stop its child .template type from being aggregatable. - super(name, true, false, false, tsi, meta, isSyntheticSource, isWithinMultiField); + super(name, IndexType.terms(true, false), false, tsi, meta, isSyntheticSource, isWithinMultiField); this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); this.textFieldType = new TextFieldMapper.TextFieldType(name, isSyntheticSource, isWithinMultiField); this.hasPositions = tsi.hasPositions(); diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java index d6c7d0c7b9aeb..f5c301e114773 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.mapper.CustomDocValuesField; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -109,14 +110,13 @@ private static class CountedKeywordFieldType extends StringFieldType { CountedKeywordFieldType( String name, - boolean isIndexed, + IndexType indexType, boolean isStored, - boolean hasDocValues, TextSearchInfo textSearchInfo, Map meta, MappedFieldType countFieldType ) { - super(name, isIndexed, isStored, hasDocValues, textSearchInfo, meta); + super(name, indexType, isStored, textSearchInfo, meta); this.countFieldType = countFieldType; } @@ -305,9 +305,8 @@ public FieldMapper build(MapperBuilderContext context) { ft, new CountedKeywordFieldType( context.buildFullName(leafName()), - isIndexed, + IndexType.terms(isIndexed, true), false, - true, new TextSearchInfo(ft, null, KEYWORD_ANALYZER, KEYWORD_ANALYZER), meta.getValue(), countFieldMapper.fieldType() diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 155cbd88cfcdd..46ebddbae790c 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -132,7 +133,13 @@ protected Parameter[] getParameters() { public static final class VersionStringFieldType extends TermBasedFieldType { private VersionStringFieldType(String name, FieldType fieldType, Map meta) { - super(name, true, false, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); + super( + name, + IndexType.terms(true, true), + false, + new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), + meta + ); } @Override From 86e5275f4919ba95bac8af29b5b9908862a7042c Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 12 Nov 2025 08:54:43 +0000 Subject: [PATCH 07/11] Use sparse indexes for dimension fields --- .../mapper/extras/ScaledFloatFieldMapper.java | 8 +- .../mapper/extras/TokenCountFieldMapper.java | 8 +- .../index/mapper/BooleanFieldMapper.java | 56 ++++++------ .../elasticsearch/index/mapper/IndexType.java | 24 ++++++ .../index/mapper/IpFieldMapper.java | 7 +- .../index/mapper/KeywordFieldMapper.java | 85 ++++++++----------- .../index/mapper/NumberFieldMapper.java | 6 +- .../fielddata/IndexFieldDataServiceTests.java | 2 +- .../plain/HalfFloatFielddataTests.java | 7 +- .../index/mapper/BooleanFieldMapperTests.java | 16 +--- .../index/mapper/BooleanFieldTypeTests.java | 11 ++- .../index/mapper/IpFieldMapperTests.java | 2 + .../index/mapper/KeywordFieldMapperTests.java | 30 ++++--- .../index/mapper/KeywordFieldTypeTests.java | 35 +++----- .../index/mapper/NumberFieldTypeTests.java | 4 +- .../index/mapper/TextFieldTypeTests.java | 10 +-- .../bucket/filter/FiltersAggregatorTests.java | 2 +- .../bucket/range/RangeAggregatorTests.java | 4 +- .../SignificantTermsAggregatorTests.java | 7 +- .../index/mapper/MapperTestCase.java | 28 ++++++ .../index/mapper/NumberFieldMapperTests.java | 2 + .../mapper/WholeNumberFieldMapperTests.java | 16 +--- .../unsignedlong/UnsignedLongFieldMapper.java | 37 ++++---- .../UnsignedLongFieldMapperTests.java | 16 +--- 24 files changed, 232 insertions(+), 191 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 40fd851b98793..4249fec6fc675 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -722,7 +722,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } long scaledValue = encode(doubleValue, scalingFactor); - NumberFieldMapper.NumberType.LONG.addFields(context.doc(), fieldType().name(), scaledValue, indexed, hasDocValues, stored); + NumberFieldMapper.NumberType.LONG.addFields( + context.doc(), + fieldType().name(), + scaledValue, + IndexType.points(indexed, hasDocValues), + stored + ); if (shouldStoreOffsets) { context.getOffSetContext().recordOffset(offsetsFieldName, scaledValue); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index 07bd713799c2a..0d77582bd9832 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -164,7 +164,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio tokenCount = countPositions(analyzer, fullPath(), value, enablePositionIncrements); } - NumberFieldMapper.NumberType.INTEGER.addFields(context.doc(), fieldType().name(), tokenCount, index, hasDocValues, store); + NumberFieldMapper.NumberType.INTEGER.addFields( + context.doc(), + fieldType().name(), + tokenCount, + IndexType.points(index, hasDocValues), + store + ); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index d59c6c31deafa..6e95417116162 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -79,7 +79,7 @@ private static BooleanFieldMapper toType(FieldMapper in) { public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter indexed; private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); private final Parameter> ignoreMalformed; private final Parameter nullValue = new Parameter<>( @@ -117,20 +117,9 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe IGNORE_MALFORMED_SETTING.get(indexSettings.getSettings()) ); this.script.precludesParameters(ignoreMalformed, nullValue); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension(), docValues::get); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, indexSettings, dimension); addScriptValidation(script, indexed, docValues); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> { - if (v && (indexed.getValue() == false || docValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + indexed.name - + "] and [" - + docValues.name - + "] are true" - ); - } - }); } public Builder dimension(boolean dimension) { @@ -152,6 +141,21 @@ protected Parameter[] getParameters() { dimension }; } + private IndexType indexType() { + if (indexed.get() && indexSettings.getIndexVersionCreated().isLegacyIndexVersion() == false) { + return IndexType.terms(true, docValues.getValue()); + } + if (docValues.get() == false) { + return IndexType.NONE; + } + if (dimension.get() + && indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) { + return IndexType.skippers(); + } + return IndexType.docValuesOnly(); + } + @Override public BooleanFieldMapper build(MapperBuilderContext context) { if (inheritDimensionParameterFromParentObject(context)) { @@ -159,9 +163,8 @@ public BooleanFieldMapper build(MapperBuilderContext context) { } MappedFieldType ft = new BooleanFieldType( context.buildFullName(leafName()), - indexed.getValue() && indexSettings.getIndexVersionCreated().isLegacyIndexVersion() == false, + indexType(), stored.getValue(), - docValues.getValue(), nullValue.getValue(), scriptValues(), meta.getValue(), @@ -215,16 +218,15 @@ public static final class BooleanFieldType extends TermBasedFieldType { public BooleanFieldType( String name, - boolean isIndexed, + IndexType indexType, boolean isStored, - boolean hasDocValues, Boolean nullValue, FieldValues scriptValues, Map meta, boolean isDimension, boolean isSyntheticSource ) { - super(name, IndexType.terms(isIndexed, hasDocValues), isStored, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + super(name, indexType, isStored, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.nullValue = nullValue; this.scriptValues = scriptValues; this.isDimension = isDimension; @@ -232,15 +234,11 @@ public BooleanFieldType( } public BooleanFieldType(String name) { - this(name, true); - } - - public BooleanFieldType(String name, boolean isIndexed) { - this(name, isIndexed, true); + this(name, IndexType.terms(true, true)); } - public BooleanFieldType(String name, boolean isIndexed, boolean hasDocValues) { - this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false, false); + public BooleanFieldType(String name, IndexType indexType) { + this(name, indexType, true, false, null, Collections.emptyMap(), false, false); } @Override @@ -601,7 +599,11 @@ private void indexValue(DocumentParserContext context, Boolean value) { context.doc().add(new StoredField(fieldType().name(), value ? "T" : "F")); } if (hasDocValues) { - context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); + if (fieldType().indexType.hasDocValuesSkipper()) { + context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), value ? 1 : 0)); + } else { + context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); + } } else { context.addToFieldNames(fieldType().name()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexType.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexType.java index 4759144e8f949..0bf9877297f8a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexType.java @@ -9,6 +9,11 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocValuesSkipIndexType; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; + import java.util.Objects; /** @@ -118,6 +123,25 @@ public static IndexType terms(boolean isIndexed, boolean hasDocValues) { return new IndexType(isIndexed, false, false, false, hasDocValues, false); } + /** + * @return a terms-based IndexType from a lucene FieldType + */ + public static IndexType terms(FieldType fieldType) { + if (fieldType.indexOptions() == IndexOptions.NONE) { + if (fieldType.docValuesType() == DocValuesType.NONE) { + return NONE; + } + if (fieldType.docValuesSkipIndexType() == DocValuesSkipIndexType.NONE) { + return docValuesOnly(); + } + return skippers(); + } + if (fieldType.docValuesType() == DocValuesType.NONE) { + return terms(true, false); + } + return terms(true, true); + } + /** * @return an IndexType with docValuesSkippers */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 0d40b7ccca297..4bd1a97e9dbe1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -106,8 +106,7 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe ); this.script.precludesParameters(nullValue, ignoreMalformed); this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); - this.indexed = Parameter.indexParam(m -> toType(m).indexed, - () -> this.dimension.get() == false && indexSettings.); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, indexSettings, dimension); addScriptValidation(script, indexed, hasDocValues); } @@ -176,7 +175,9 @@ private IndexType indexType() { if (indexSettings.getIndexVersionCreated().isLegacyIndexVersion()) { return hasDocValues.get() ? IndexType.archivedPoints() : IndexType.NONE; } - if (dimension.get()) { + if (dimension.get() + && indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) { return IndexType.skippers(); } return IndexType.points(indexed.get(), hasDocValues.get()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index cef252abc45ce..448c4ebedf750 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -396,10 +396,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } return new KeywordFieldType( context.buildFullName(leafName()), - fieldType, + IndexType.terms(fieldType), + new TextSearchInfo(fieldType, similarity.get(), searchAnalyzer, quoteAnalyzer), normalizer, - searchAnalyzer, - quoteAnalyzer, this, context.isSourceSynthetic() ); @@ -408,25 +407,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { FieldType fieldtype = resolveFieldType( - forceDocValuesSkipper || (this.dimension.get() && this.indexed.get() == false), - hasDocValues.get(), - indexSettings, + forceDocValuesSkipper, context.buildFullName(leafName()) ); - fieldtype.setOmitNorms(this.hasNorms.getValue() == false); - fieldtype.setStored(this.stored.getValue()); - fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); - if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES) == false) { - // NOTE: override index options only if we are not using a sparse doc values index (and we use an inverted index) - fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); - } - if (fieldtype.equals(Defaults.FIELD_TYPE)) { - // deduplicate in the common default case to save some memory - fieldtype = Defaults.FIELD_TYPE; - } - if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) { - fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES; - } super.hasScript = script.get() != null; super.onScriptError = onScriptError.getValue(); @@ -449,24 +432,43 @@ public KeywordFieldMapper build(MapperBuilderContext context) { ); } - private static FieldType resolveFieldType( + private FieldType resolveFieldType( final boolean forceDocValuesSkipper, - boolean hasDocValues, - final IndexSettings indexSettings, final String fullFieldName ) { - if (forceDocValuesSkipper || shouldUseDocValuesSkipper(hasDocValues, indexSettings, fullFieldName)) { - return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); + FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); + if (forceDocValuesSkipper || shouldUseHostnameSkipper(fullFieldName) || shouldUseDimensionSkipper()) { + fieldtype = new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); } - return new FieldType(Defaults.FIELD_TYPE); + fieldtype.setOmitNorms(this.hasNorms.getValue() == false); + fieldtype.setStored(this.stored.getValue()); + fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); + if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES) == false) { + // NOTE: override index options only if we are not using a sparse doc values index (and we use an inverted index) + fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); + } + if (fieldtype.equals(Defaults.FIELD_TYPE)) { + // deduplicate in the common default case to save some memory + fieldtype = Defaults.FIELD_TYPE; + } + if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) { + fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES; + } + return fieldtype; + } + + private boolean shouldUseDimensionSkipper() { + return hasDocValues.get() + && dimension.get() + && indexed.get() == false + && indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); } - private static boolean shouldUseDocValuesSkipper( - final boolean hasDocValues, - final IndexSettings indexSettings, + private boolean shouldUseHostnameSkipper( final String fullFieldName ) { - return hasDocValues + return hasDocValues.get() && indexSettings.useDocValuesSkipper() && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.SKIPPERS_ENABLED_BY_DEFAULT) && IndexMode.LOGSDB.equals(indexSettings.getMode()) @@ -491,25 +493,20 @@ public static final class KeywordFieldType extends TextFamilyFieldType { private final boolean eagerGlobalOrdinals; private final FieldValues scriptValues; private final boolean isDimension; - private final boolean hasDocValuesSkipper; public KeywordFieldType( String name, - FieldType fieldType, + IndexType indexType, + TextSearchInfo textSearchInfo, NamedAnalyzer normalizer, - NamedAnalyzer searchAnalyzer, - NamedAnalyzer quoteAnalyzer, Builder builder, boolean isSyntheticSource ) { super( name, - IndexType.terms( - fieldType.indexOptions() != IndexOptions.NONE && builder.indexCreatedVersion.isLegacyIndexVersion() == false, - builder.hasDocValues.get() - ), - fieldType.stored(), - textSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, quoteAnalyzer), + indexType, + builder.stored.get(), + textSearchInfo, builder.meta.getValue(), isSyntheticSource, builder.isWithinMultiField @@ -524,7 +521,6 @@ public KeywordFieldType( this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); - this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name) { @@ -539,7 +535,6 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.hasDocValuesSkipper = false; } public KeywordFieldType(String name, FieldType fieldType) { @@ -558,7 +553,6 @@ public KeywordFieldType(String name, FieldType fieldType) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -577,7 +571,6 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.hasDocValuesSkipper = false; } @Override @@ -1042,10 +1035,6 @@ public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } - public boolean hasDocValuesSkipper() { - return hasDocValuesSkipper; - } - @Override public Query automatonQuery( Supplier automatonSupplier, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index de87a4d1cca75..0e4c6c7aeefe2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -166,7 +166,12 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, IndexSetti XContentBuilder::field, Objects::toString ).acceptsNull(); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { + if (dimension.get()) { + return indexSettings.useDocValuesSkipper() == false || + indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); + } if (indexSettings.getMode() == IndexMode.TIME_SERIES) { var metricType = getMetric().getValue(); return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE; @@ -174,7 +179,6 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, IndexSetti return true; } }); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE, MetricType.COUNTER).addValidator(v -> { if (v != null && hasDocValues.getValue() == false) { diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 5431e3925a1d7..d4e838493afbd 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -341,7 +341,7 @@ public void testRequireDocValuesOnDoubles() { public void testRequireDocValuesOnBools() { doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field")); doTestRequireDocValues( - new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false, false) + new BooleanFieldMapper.BooleanFieldType("field", IndexType.terms(true, false), false, null, null, Collections.emptyMap(), false, false) ); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java index 02a9d7dc59940..79e16220ecc57 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues.Doubles; import org.elasticsearch.index.fielddata.ScriptDocValues.DoublesSupplier; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.script.field.DelegateDocValuesField; @@ -34,7 +35,7 @@ public void testSingleValued() throws IOException { // we need the default codec to check for singletons IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setCodec(TestUtil.getDefaultCodec())); LuceneDocument doc = new LuceneDocument(); - NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 3f, false, true, false); + NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 3f, IndexType.points(false, true), false); w.addDocument(doc); final DirectoryReader dirReader = DirectoryReader.open(w); LeafReader reader = getOnlyLeafReader(dirReader); @@ -54,8 +55,8 @@ public void testMultiValued() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); LuceneDocument doc = new LuceneDocument(); - NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 3f, false, true, false); - NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 2f, false, true, false); + NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 3f, IndexType.points(false, true), false); + NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 2f, IndexType.points(false, true), false); w.addDocument(doc); final DirectoryReader dirReader = DirectoryReader.open(w); LeafReader reader = getOnlyLeafReader(dirReader); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 76df9b6303109..fbee1f0bee54e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -226,6 +226,8 @@ public void testDimension() throws IOException { assertDimension(true, BooleanFieldMapper.BooleanFieldType::isDimension); assertDimension(false, BooleanFieldMapper.BooleanFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionIndexedAndDocvalues() { @@ -236,7 +238,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } { @@ -246,17 +248,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java index 38f73cd9f681d..3c521b7e55caf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java @@ -40,11 +40,11 @@ public void testTermQuery() { assertEquals(new TermQuery(new Term("field", "T")), ft.termQuery("true", MOCK_CONTEXT)); assertEquals(new TermQuery(new Term("field", "F")), ft.termQuery("false", MOCK_CONTEXT)); - MappedFieldType ft2 = new BooleanFieldMapper.BooleanFieldType("field", false); + MappedFieldType ft2 = new BooleanFieldMapper.BooleanFieldType("field", IndexType.terms(false, true)); assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 1), ft2.termQuery("true", MOCK_CONTEXT)); assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 0), ft2.termQuery("false", MOCK_CONTEXT)); - MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false); + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", IndexType.NONE); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("true", MOCK_CONTEXT)); assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); } @@ -54,11 +54,11 @@ public void testRangeQuery() { Query expected = new TermRangeQuery("field", BooleanFieldMapper.Values.FALSE, BooleanFieldMapper.Values.TRUE, true, true); assertEquals(expected, ft.rangeQuery("false", "true", true, true, null, null, null, MOCK_CONTEXT)); - ft = new BooleanFieldMapper.BooleanFieldType("field", false); + ft = new BooleanFieldMapper.BooleanFieldType("field", IndexType.points(false, true)); expected = SortedNumericDocValuesField.newSlowRangeQuery("field", 0, 1); assertEquals(expected, ft.rangeQuery("false", "true", true, true, null, null, null, MOCK_CONTEXT)); - MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false); + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", IndexType.NONE); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> unsearchable.rangeQuery("false", "true", true, true, null, null, null, MOCK_CONTEXT) @@ -75,10 +75,9 @@ public void testFetchSourceValue() throws IOException { MappedFieldType nullFieldType = new BooleanFieldMapper.BooleanFieldType( "field", - true, + IndexType.points(true, true), false, true, - true, null, Collections.emptyMap(), false, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 137fceb2e6370..dc98c26969b9c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -224,6 +224,8 @@ public void testDimension() throws IOException { assertDimension(true, IpFieldMapper.IpFieldType::isDimension); assertDimension(false, IpFieldMapper.IpFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionIndexedAndDocvalues() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index b65c66e6fb197..29bc8c47f57b4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -427,6 +427,8 @@ public void testDimension() throws IOException { assertDimension(true, KeywordFieldMapper.KeywordFieldType::isDimension); assertDimension(false, KeywordFieldMapper.KeywordFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionAndIgnoreAbove() throws IOException { @@ -967,9 +969,9 @@ public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasTerms()); } } @@ -988,7 +990,7 @@ public void testFieldTypeDefault_StandardMode() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().indexType().hasTerms()); - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); } public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { @@ -1005,7 +1007,7 @@ public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("hostname"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().indexType().hasTerms()); - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType.hasDocValuesSkipper()); } public void testFieldTypeDefault_ConfiguredIndexedWithSettingOverride() throws IOException { @@ -1023,9 +1025,9 @@ public void testFieldTypeDefault_ConfiguredIndexedWithSettingOverride() throws I final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasTerms()); } } @@ -1045,9 +1047,9 @@ public void testFieldTypeDefault_ConfiguredIndexedWithoutSettingOverride() throw final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasTerms()); } } @@ -1067,9 +1069,9 @@ public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasTerms()); } } @@ -1088,7 +1090,7 @@ public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().indexType().hasTerms()); - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); } public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IOException { @@ -1107,9 +1109,9 @@ public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IO final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasOnlyDocValues()); } } @@ -1130,7 +1132,7 @@ public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertFalse(mapper.fieldType().hasDocValues()); assertThat(mapper.fieldType().indexType(), equalTo(IndexType.NONE)); - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); } public void testValueIsStoredWhenItExceedsIgnoreAboveAndFieldIsNotAMultiField() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 8e2e29343a195..19987a7ae8e39 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -327,9 +327,8 @@ public void testIgnoreAboveIndexLevelSetting() { KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -359,9 +358,8 @@ public void testIgnoreAboveIsSetReturnsTrueWhenIgnoreAboveIsGiven() { KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -390,9 +388,8 @@ public void testIgnoreAboveIsSetReturnsFalseWhenIgnoreAboveIsNotGiven() { KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -422,9 +419,8 @@ public void testIgnoreAboveIsSetReturnsFalseWhenIgnoreAboveIsGivenButItsTheSameA KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -454,9 +450,8 @@ public void testIgnoreAboveIsSetReturnsFalseWhenIgnoreAboveIsGivenButItsTheSameA KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -486,9 +481,8 @@ public void testIgnoreAboveIsSetReturnsTrueWhenIgnoreAboveIsGivenAsLogsdbDefault KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -518,9 +512,8 @@ public void testIgnoreAboveIsSetReturnsTrueWhenIgnoreAboveIsConfiguredAtIndexLev KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 735a61a25eabe..29e090cec76b1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -679,7 +679,7 @@ public void doTestDocValueRangeQueries(NumberType type, Supplier valueSu final int numDocs = TestUtil.nextInt(random(), 100, 500); for (int i = 0; i < numDocs; ++i) { final LuceneDocument doc = new LuceneDocument(); - type.addFields(doc, "foo", valueSupplier.get(), true, true, false); + type.addFields(doc, "foo", valueSupplier.get(), IndexType.points(true, true), false); w.addDocument(doc); } DirectoryReader reader = DirectoryReader.open(w); @@ -733,7 +733,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS final int numDocs = TestUtil.nextInt(random(), 100, 500); for (int i = 0; i < numDocs; ++i) { final LuceneDocument doc = new LuceneDocument(); - type.addFields(doc, "field", valueSupplier.get(), true, true, false); + type.addFields(doc, "field", valueSupplier.get(), IndexType.points(true, true), false); w.addDocument(doc); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 4787b444d732f..95921cd410368 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -353,9 +353,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -404,9 +403,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index 0af4a8b31cd76..69cfa0bcd240f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -1486,7 +1486,7 @@ public void testDocValuesFieldExistsForNumber() throws IOException { NumberFieldMapper.NumberFieldType ft = new NumberFieldMapper.NumberFieldType("f", numberType); docValuesFieldExistsTestCase(new ExistsQueryBuilder("f"), ft, true, i -> { final LuceneDocument document = new LuceneDocument(); - numberType.addFields(document, "f", i, true, true, false); + numberType.addFields(document, "f", i, IndexType.points(true, true), false); return document; }); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index d58b72f788c03..b71cf00859700 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -235,7 +235,7 @@ public void testFloatRangesExclusiveEndpoint() throws IOException { MappedFieldType field = new NumberFieldMapper.NumberFieldType(fieldName, NumberType.FLOAT); testCase(iw -> { LuceneDocument doc = new LuceneDocument(); - NumberType.FLOAT.addFields(doc, fieldName, 0.04F, false, true, false); + NumberType.FLOAT.addFields(doc, fieldName, 0.04F, IndexType.points(false, true), false); iw.addDocument(doc); }, result -> { InternalRange range = (InternalRange) result; @@ -259,7 +259,7 @@ public void testHalfFloatRangesExclusiveEndpoint() throws IOException { MappedFieldType field = new NumberFieldMapper.NumberFieldType(fieldName, NumberType.HALF_FLOAT); testCase(iw -> { LuceneDocument doc = new LuceneDocument(); - NumberType.HALF_FLOAT.addFields(doc, fieldName, 0.0152F, false, true, false); + NumberType.HALF_FLOAT.addFields(doc, fieldName, 0.0152F, IndexType.points(false, true), false); iw.addDocument(doc); }, result -> { InternalRange range = (InternalRange) result; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index 504b757ffa42f..f6806111050ae 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.BinaryFieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; @@ -259,13 +260,13 @@ public void testNumericSignificance() throws IOException { for (int i = 0; i < 10; i++) { LuceneDocument doc = new LuceneDocument(); if (i % 2 == 0) { - NumberType.LONG.addFields(doc, "long_field", ODD_VALUE, true, true, false); + NumberType.LONG.addFields(doc, "long_field", ODD_VALUE, IndexType.points(true, true), false); doc.add(new Field("text", "odd", TextFieldMapper.Defaults.FIELD_TYPE)); } else { - NumberType.LONG.addFields(doc, "long_field", EVEN_VALUE, true, true, false); + NumberType.LONG.addFields(doc, "long_field", EVEN_VALUE, IndexType.points(true, true), false); doc.add(new Field("text", "even", TextFieldMapper.Defaults.FIELD_TYPE)); } - NumberType.LONG.addFields(doc, "long_field", COMMON_VALUE, true, true, false); + NumberType.LONG.addFields(doc, "long_field", COMMON_VALUE, IndexType.points(true, true), false); w.addDocument(doc); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 7e32e5e2b1c00..97ccb4b7ce7c6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -12,6 +12,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexWriterConfig; @@ -387,6 +388,33 @@ protected void assertDimension(boolean isDimension, Function che @SuppressWarnings("unchecked") // Syntactic sugar in tests T fieldType = (T) mapperService.fieldType("field"); assertThat(checker.apply(fieldType), equalTo(isDimension)); + + Settings settings = Settings.builder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true).build(); + mapperService = createMapperService(settings, fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", isDimension); + })); + + assertThat( + mapperService.fieldType("field").indexType().hasDocValuesSkipper(), + equalTo(mapperService.getIndexSettings().useDocValuesSkipper() && isDimension) + ); + } + + public void assertDimensionIndexing() throws IOException { + Settings settings = Settings.builder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true).build(); + MapperService mapperService = createMapperService(settings, fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true); + })); + ParsedDocument doc = mapperService.documentMapper().parse(source(this::writeField)); + IndexableField field = doc.rootDoc().getField("field"); + assertEquals(field.fieldType().docValuesSkipIndexType() == DocValuesSkipIndexType.RANGE, + mapperService.getIndexSettings().useDocValuesSkipper()); + assertEquals(field.fieldType().indexOptions() == IndexOptions.NONE, + mapperService.getIndexSettings().useDocValuesSkipper()); + assertEquals(field.fieldType().pointDimensionCount() == 0, + mapperService.getIndexSettings().useDocValuesSkipper()); } protected void assertMetricType(String metricType, Function> checker) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 2f16534908b2e..310d891947fb2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -254,6 +254,8 @@ public void testDimension() throws IOException { // dimension = true is allowed assertDimension(true, NumberFieldMapper.NumberFieldType::isDimension); + + assertDimensionIndexing(); } public void testMetricType() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java index 64a7868607cf1..19207900485d4 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java @@ -39,6 +39,8 @@ public void testDimension() throws IOException { assertDimension(true, NumberFieldMapper.NumberFieldType::isDimension); assertDimension(false, NumberFieldMapper.NumberFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionIndexedAndDocvalues() { @@ -49,7 +51,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } { @@ -59,17 +61,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index cc8b11e7eedb2..a2e6c0895159e 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -123,7 +123,12 @@ public Builder(String name, IndexSettings indexSettings) { XContentBuilder::field, Objects::toString ).acceptsNull(); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { + if (dimension.get()) { + return indexSettings.useDocValuesSkipper() == false || + indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); + } if (indexSettings.getMode() == IndexMode.TIME_SERIES) { var metricType = getMetric().getValue(); return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE; @@ -131,19 +136,6 @@ public Builder(String name, IndexSettings indexSettings) { return true; } }); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + indexed.name - + "] and [" - + hasDocValues.name - + "] are true" - ); - } - }); this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE, MetricType.COUNTER).addValidator(v -> { if (v != null && hasDocValues.getValue() == false) { @@ -198,6 +190,17 @@ Number parsedNullValue() { return parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); } + private IndexType indexType() { + if (indexSettings.useDocValuesSkipper() + && indexed.get() == false + && hasDocValues.get() + && dimension.get() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) { + return IndexType.skippers(); + } + return IndexType.points(indexed.get(), hasDocValues.get()); + } + @Override public UnsignedLongFieldMapper build(MapperBuilderContext context) { if (inheritDimensionParameterFromParentObject(context)) { @@ -205,7 +208,7 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) { } UnsignedLongFieldType fieldType = new UnsignedLongFieldType( context.buildFullName(leafName()), - IndexType.points(indexed.get(), hasDocValues.get()), + indexType(), stored.getValue(), parsedNullValue(), meta.getValue(), @@ -767,7 +770,11 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (indexed && hasDocValues) { fields.add(new LongField(fieldType().name(), numericValue, Field.Store.NO)); } else if (hasDocValues) { - fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue)); + if (fieldType().indexType().hasDocValuesSkipper()) { + fields.add(SortedNumericDocValuesField.indexedField(fieldType().name(), numericValue)); + } else { + fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue)); + } } else if (indexed) { fields.add(new LongPoint(fieldType().name(), numericValue)); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index ea3e4df43d615..d3e478db66c03 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -228,6 +228,8 @@ public void testDimension() throws IOException { assertDimension(true, UnsignedLongFieldMapper.UnsignedLongFieldType::isDimension); assertDimension(false, UnsignedLongFieldMapper.UnsignedLongFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionIndexedAndDocvalues() { @@ -238,7 +240,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } { @@ -248,17 +250,7 @@ public void testDimensionIndexedAndDocvalues() { }))); assertThat( e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") + containsString("Field [time_series_dimension] requires that [doc_values] is true") ); } } From ddd964741c0a63382a429f91e7cabf49aa97b8b8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 12 Nov 2025 10:16:09 +0000 Subject: [PATCH 08/11] iter --- .../index/mapper/KeywordFieldMapper.java | 14 +++----------- .../index/mapper/NumberFieldMapper.java | 12 +++--------- .../index/mapper/TimeSeriesParams.java | 3 +-- .../fielddata/IndexFieldDataServiceTests.java | 11 ++++++++++- .../index/mapper/BooleanFieldMapperTests.java | 10 ++-------- .../index/mapper/IpFieldMapperTests.java | 10 ++-------- .../elasticsearch/index/mapper/MapperTestCase.java | 11 +++++------ .../index/mapper/WholeNumberFieldMapperTests.java | 10 ++-------- .../unsignedlong/UnsignedLongFieldMapper.java | 4 ++-- .../unsignedlong/UnsignedLongFieldMapperTests.java | 10 ++-------- 10 files changed, 32 insertions(+), 63 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index aefbf4a1bebb5..bda26bebad7f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -406,10 +406,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { - FieldType fieldtype = resolveFieldType( - forceDocValuesSkipper, - context.buildFullName(leafName()) - ); + FieldType fieldtype = resolveFieldType(forceDocValuesSkipper, context.buildFullName(leafName())); super.hasScript = script.get() != null; super.onScriptError = onScriptError.getValue(); @@ -432,10 +429,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) { ); } - private FieldType resolveFieldType( - final boolean forceDocValuesSkipper, - final String fullFieldName - ) { + private FieldType resolveFieldType(final boolean forceDocValuesSkipper, final String fullFieldName) { FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); if (forceDocValuesSkipper || shouldUseHostnameSkipper(fullFieldName) || shouldUseDimensionSkipper()) { fieldtype = new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); @@ -465,9 +459,7 @@ private boolean shouldUseDimensionSkipper() { && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); } - private boolean shouldUseHostnameSkipper( - final String fullFieldName - ) { + private boolean shouldUseHostnameSkipper(final String fullFieldName) { return hasDocValues.get() && indexSettings.useDocValuesSkipper() && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.SKIPPERS_ENABLED_BY_DEFAULT) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 0e4c6c7aeefe2..cfbab8887bdc7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -169,8 +169,8 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, IndexSetti this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { if (dimension.get()) { - return indexSettings.useDocValuesSkipper() == false || - indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); + return indexSettings.useDocValuesSkipper() == false + || indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); } if (indexSettings.getMode() == IndexMode.TIME_SERIES) { var metricType = getMetric().getValue(); @@ -1567,13 +1567,7 @@ public abstract Query rangeQuery( * @param indexType an IndexType describing the index structures to be added * @param stored whether or not the field is stored */ - public abstract void addFields( - LuceneDocument document, - String name, - Number value, - IndexType indexType, - boolean stored - ); + public abstract void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored); /** * For a given {@code Number}, returns the sortable long representation that will be stored in the doc values. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java index 1db2221be088c..66af7bb97f59c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java @@ -90,8 +90,7 @@ public static FieldMapper.Parameter metricParam(Function dimensionParam(Function initializer, BooleanSupplier hasDocValues) { - return FieldMapper.Parameter.boolParam(TIME_SERIES_DIMENSION_PARAM, false, initializer, false) - .addValidator(v -> { + return FieldMapper.Parameter.boolParam(TIME_SERIES_DIMENSION_PARAM, false, initializer, false).addValidator(v -> { if (v && (hasDocValues.getAsBoolean() == false)) { throw new IllegalArgumentException( "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] requires that [doc_values] is true" diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index d4e838493afbd..4fd688557025f 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -341,7 +341,16 @@ public void testRequireDocValuesOnDoubles() { public void testRequireDocValuesOnBools() { doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field")); doTestRequireDocValues( - new BooleanFieldMapper.BooleanFieldType("field", IndexType.terms(true, false), false, null, null, Collections.emptyMap(), false, false) + new BooleanFieldMapper.BooleanFieldType( + "field", + IndexType.terms(true, false), + false, + null, + null, + Collections.emptyMap(), + false, + false + ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index fbee1f0bee54e..9be4ff52d39f4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -236,20 +236,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index dc98c26969b9c..4874442b9c5a0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -234,20 +234,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index fa8094cb9eb0b..629282256050d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -407,14 +407,13 @@ public void assertDimensionIndexing() throws IOException { minimalMapping(b); b.field("time_series_dimension", true); })); + assumeTrue("Skippers disabled by feature flag", mapperService.getIndexSettings().useDocValuesSkipper()); + ParsedDocument doc = mapperService.documentMapper().parse(source(this::writeField)); IndexableField field = doc.rootDoc().getField("field"); - assertEquals(field.fieldType().docValuesSkipIndexType() == DocValuesSkipIndexType.RANGE, - mapperService.getIndexSettings().useDocValuesSkipper()); - assertEquals(field.fieldType().indexOptions() == IndexOptions.NONE, - mapperService.getIndexSettings().useDocValuesSkipper()); - assertEquals(field.fieldType().pointDimensionCount() == 0, - mapperService.getIndexSettings().useDocValuesSkipper()); + assertSame(DocValuesSkipIndexType.RANGE, field.fieldType().docValuesSkipIndexType()); + assertSame(IndexOptions.NONE, field.fieldType().indexOptions()); + assertEquals(0, field.fieldType().pointDimensionCount()); } protected void assertMetricType(String metricType, Function> checker) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java index 19207900485d4..42f757e73018c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java @@ -49,20 +49,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index a2e6c0895159e..59da232924c09 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -126,8 +126,8 @@ public Builder(String name, IndexSettings indexSettings) { this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { if (dimension.get()) { - return indexSettings.useDocValuesSkipper() == false || - indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); + return indexSettings.useDocValuesSkipper() == false + || indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); } if (indexSettings.getMode() == IndexMode.TIME_SERIES) { var metricType = getMetric().getValue(); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index d3e478db66c03..c852f3daedb39 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -238,20 +238,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [doc_values] is true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } From 57b0a611c14c1b349bd23bc05f3753092f44f51e Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 12 Nov 2025 10:48:00 +0000 Subject: [PATCH 09/11] precommit --- .../esql/ValuesSourceReaderBenchmark.java | 69 +++++++++---------- .../extras/MatchOnlyTextFieldTypeTests.java | 11 ++- .../percolator/CandidateQueryTests.java | 11 ++- .../index/mapper/size/SizeFieldMapper.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 10 ++- .../index/mapper/KeywordFieldTypeTests.java | 4 +- .../ValueSourceReaderTypeConversionTests.java | 11 +-- .../read/ValuesSourceReaderOperatorTests.java | 10 +-- .../planner/EsPhysicalOperationProviders.java | 7 +- 9 files changed, 61 insertions(+), 74 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java index 4b6b54979509c..17795bfdc755c 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java @@ -26,7 +26,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; @@ -227,50 +226,44 @@ private static BlockLoader blockLoader(String name) { break; } ft.freeze(); - return new KeywordFieldMapper.KeywordFieldType( - w.name, - ft, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(ft.docValuesType() != DocValuesType.NONE), - syntheticSource - ).blockLoader(new MappedFieldType.BlockLoaderContext() { - @Override - public String indexName() { - return "benchmark"; - } + return new KeywordFieldMapper.KeywordFieldType(w.name, ft, syntheticSource).blockLoader( + new MappedFieldType.BlockLoaderContext() { + @Override + public String indexName() { + return "benchmark"; + } - @Override - public IndexSettings indexSettings() { - throw new UnsupportedOperationException(); - } + @Override + public IndexSettings indexSettings() { + throw new UnsupportedOperationException(); + } - @Override - public MappedFieldType.FieldExtractPreference fieldExtractPreference() { - return MappedFieldType.FieldExtractPreference.NONE; - } + @Override + public MappedFieldType.FieldExtractPreference fieldExtractPreference() { + return MappedFieldType.FieldExtractPreference.NONE; + } - @Override - public SearchLookup lookup() { - throw new UnsupportedOperationException(); - } + @Override + public SearchLookup lookup() { + throw new UnsupportedOperationException(); + } - @Override - public Set sourcePaths(String name) { - return Set.of(name); - } + @Override + public Set sourcePaths(String name) { + return Set.of(name); + } - @Override - public String parentField(String field) { - throw new UnsupportedOperationException(); - } + @Override + public String parentField(String field) { + throw new UnsupportedOperationException(); + } - @Override - public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { - return FieldNamesFieldMapper.FieldNamesFieldType.get(true); + @Override + public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { + return FieldNamesFieldMapper.FieldNamesFieldType.get(true); + } } - }); + ); } throw new IllegalArgumentException("can't read [" + name + "]"); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java index 42ee978239294..036cb55fa03cf 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java @@ -42,6 +42,7 @@ import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappingParserContext; @@ -295,9 +296,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -344,9 +344,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index ff321303b56c0..16ee2ce634b5d 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -79,6 +79,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DocumentParserContext; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; @@ -301,7 +302,7 @@ public void testDuel() throws Exception { document.add(new TextField(entry.getKey(), value, Field.Store.NO)); } for (Integer intValue : intValues) { - NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", intValue, true, true, false); + NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", intValue, IndexType.points(true, false), true); } MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer()); duelRun(queryStore, memoryIndex, shardSearcher); @@ -424,7 +425,13 @@ public void testDuel2() throws Exception { } for (int[] range : ranges) { - NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", between(range[0], range[1]), true, true, false); + NumberFieldMapper.NumberType.INTEGER.addFields( + document, + "int_field", + between(range[0], range[1]), + IndexType.points(true, false), + true + ); logger.info("Test with document: {}" + document); MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer()); duelRun(queryStore, memoryIndex, shardSearcher); diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index ccd0b16666a6a..b8a0be5b6629e 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -104,7 +104,7 @@ public void postParse(DocumentParserContext context) { return; } final int value = context.sourceToParse().source().length(); - NumberType.INTEGER.addFields(context.doc(), fullPath(), value, true, true, true); + NumberType.INTEGER.addFields(context.doc(), fullPath(), value, IndexType.points(true, true), true); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index bda26bebad7f3..e8759466c9ef4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -314,6 +314,10 @@ public boolean hasDocValues() { return this.hasDocValues.get(); } + public SimilarityProvider similarity() { + return this.similarity.get(); + } + public Builder dimension(boolean dimension) { this.dimension.setValue(dimension); return this; @@ -529,14 +533,14 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.isDimension = false; } - public KeywordFieldType(String name, FieldType fieldType) { + public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSource) { super( name, IndexType.terms(fieldType.indexOptions() != IndexOptions.NONE, false), - false, + fieldType.stored(), textSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap(), - false, + isSyntheticSource, false ); this.normalizer = Lucene.KEYWORD_ANALYZER; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 19987a7ae8e39..8c2f19d02e046 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -143,7 +143,7 @@ public void testExistsQuery() { { FieldType fieldType = new FieldType(); fieldType.setOmitNorms(false); - KeywordFieldType ft = new KeywordFieldType("field", fieldType); + KeywordFieldType ft = new KeywordFieldType("field", fieldType, false); // updated in #130531 so that a field that is neither indexed nor has doc values will generate a TermQuery // to avoid ISE from FieldExistsQuery assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(MOCK_CONTEXT)); @@ -527,7 +527,7 @@ public void testIgnoreAboveIsSetReturnsTrueWhenIgnoreAboveIsConfiguredAtIndexLev public void testIgnoreAboveIsSetReturnsFalseForNonPrimaryConstructor() { // given KeywordFieldType fieldType1 = new KeywordFieldType("field"); - KeywordFieldType fieldType2 = new KeywordFieldType("field", mock(FieldType.class)); + KeywordFieldType fieldType2 = new KeywordFieldType("field", mock(FieldType.class), false); KeywordFieldType fieldType3 = new KeywordFieldType("field", true, true, Collections.emptyMap()); KeywordFieldType fieldType4 = new KeywordFieldType("field", mock(NamedAnalyzer.class)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index 7a0e5c1c6cd83..dca14c8dd0d2c 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; @@ -1401,15 +1400,7 @@ private KeywordFieldMapper.KeywordFieldType storedKeywordField(String name) { ft.setDocValuesType(DocValuesType.NONE); ft.setStored(true); ft.freeze(); - return new KeywordFieldMapper.KeywordFieldType( - name, - ft, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(false), - true // TODO randomize - load from stored keyword fields if stored even in synthetic source - ); + return new KeywordFieldMapper.KeywordFieldType(name, ft, false); } @AwaitsFix(bugUrl = "Get working for multiple indices") diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index 9ee061b6e1d6e..cf4132fb52180 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -1577,15 +1577,7 @@ private KeywordFieldMapper.KeywordFieldType storedKeywordField(String name) { ft.setDocValuesType(DocValuesType.NONE); ft.setStored(true); ft.freeze(); - return new KeywordFieldMapper.KeywordFieldType( - name, - ft, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(false), - true // TODO randomize - load from stored keyword fields if stored even in synthetic source - ); + return new KeywordFieldMapper.KeywordFieldType(name, ft, false); } private TextFieldMapper.TextFieldType storedTextField(String name) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index f15c831acf5a8..e708eae794be7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -41,10 +41,12 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -242,9 +244,8 @@ static MappedFieldType createUnmappedFieldType(String name, DefaultShardContext builder.indexed(false); return new KeywordFieldMapper.KeywordFieldType( name, - UNMAPPED_FIELD_TYPE, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, + IndexType.terms(false, false), + new TextSearchInfo(UNMAPPED_FIELD_TYPE, builder.similarity(), Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Lucene.KEYWORD_ANALYZER, builder, context.ctx.isSourceSynthetic() From 667953ebf70d002fb947f3e014ddc674bea2d519 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 12 Nov 2025 12:59:51 +0000 Subject: [PATCH 10/11] iter --- .../java/org/elasticsearch/index/mapper/KeywordFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e8759466c9ef4..4bca305eb5da9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -536,7 +536,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSource) { super( name, - IndexType.terms(fieldType.indexOptions() != IndexOptions.NONE, false), + IndexType.terms(fieldType), fieldType.stored(), textSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap(), From 38ee47ad02e445ed4abee33c0738a5f194fe450a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 12 Nov 2025 13:40:18 +0000 Subject: [PATCH 11/11] iter --- .../org/elasticsearch/percolator/CandidateQueryTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index 16ee2ce634b5d..1268705436d28 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -302,7 +302,7 @@ public void testDuel() throws Exception { document.add(new TextField(entry.getKey(), value, Field.Store.NO)); } for (Integer intValue : intValues) { - NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", intValue, IndexType.points(true, false), true); + NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", intValue, IndexType.points(true, true), false); } MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer()); duelRun(queryStore, memoryIndex, shardSearcher); @@ -429,8 +429,8 @@ public void testDuel2() throws Exception { document, "int_field", between(range[0], range[1]), - IndexType.points(true, false), - true + IndexType.points(true, true), + false ); logger.info("Test with document: {}" + document); MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer());