diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index e1fcfdec7b039..03e4b0f533b53 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -212,6 +213,16 @@ public void testPickingBackingIndicesNanoTimestamp() throws Exception { } } + private static final IndexSettings DEFAULT_INDEX_SETTINGS = new IndexSettings( + IndexMetadata.builder("_na_") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(), + Settings.EMPTY + ); + @Before public void setup() throws Exception { testThreadPool = new TestThreadPool(getTestName()); @@ -224,8 +235,7 @@ public void setup() throws Exception { DateFieldMapper.Resolution.MILLISECONDS, null, ScriptCompiler.NONE, - false, - IndexVersion.current() + DEFAULT_INDEX_SETTINGS ).build(MapperBuilderContext.root(false, false)); RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc", ObjectMapper.Defaults.SUBOBJECTS); root.add( @@ -234,9 +244,8 @@ public void setup() throws Exception { DateFieldMapper.Resolution.MILLISECONDS, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, - true, - IndexVersion.current() - ) + DEFAULT_INDEX_SETTINGS + ).ignoreMalformed(true) ); MetadataFieldMapper dtfm = DataStreamTestHelper.getDataStreamTimestampFieldMapper(); Mapping mapping = new Mapping( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index c6810ed0af4cf..e576169902ada 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -38,7 +38,6 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -287,47 +286,25 @@ public static final class Builder extends FieldMapper.Builder { private final Resolution resolution; private final IndexVersion indexCreatedVersion; private final ScriptCompiler scriptCompiler; - private final IndexMode indexMode; - private final IndexSortConfig indexSortConfig; - private final boolean useDocValuesSkipper; + private final IndexSettings indexSettings; public Builder( String name, Resolution resolution, DateFormatter dateFormatter, ScriptCompiler scriptCompiler, - boolean ignoreMalformedByDefault, - IndexVersion indexCreatedVersion - ) { - this( - name, - resolution, - dateFormatter, - scriptCompiler, - ignoreMalformedByDefault, - IndexMode.STANDARD, - null, - indexCreatedVersion, - false - ); - } - - public Builder( - String name, - Resolution resolution, - DateFormatter dateFormatter, - ScriptCompiler scriptCompiler, - boolean ignoreMalformedByDefault, - IndexMode indexMode, - IndexSortConfig indexSortConfig, - IndexVersion indexCreatedVersion, - boolean useDocValuesSkipper + IndexSettings indexSettings ) { super(name); this.resolution = resolution; - this.indexCreatedVersion = indexCreatedVersion; + this.indexCreatedVersion = indexSettings.getIndexVersionCreated(); this.scriptCompiler = Objects.requireNonNull(scriptCompiler); - this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault); + this.ignoreMalformed = Parameter.boolParam( + "ignore_malformed", + true, + m -> toType(m).ignoreMalformed, + FieldMapper.IGNORE_MALFORMED_SETTING.get(indexSettings.getSettings()) + ); this.script.precludesParameters(nullValue, ignoreMalformed); addScriptValidation(script, index, docValues); @@ -345,9 +322,12 @@ public Builder( this.format.setValue(dateFormatter.pattern()); this.locale.setValue(dateFormatter.locale()); } - this.indexMode = indexMode; - this.indexSortConfig = indexSortConfig; - this.useDocValuesSkipper = useDocValuesSkipper; + this.indexSettings = indexSettings; + } + + public Builder ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed.setValue(ignoreMalformed); + return this; } DateFormatter buildFormatter() { @@ -419,14 +399,7 @@ private Long parseNullValue(DateFieldType fieldType) { } private IndexType indexType(String fullFieldName) { - boolean hasDocValuesSkipper = shouldUseDocValuesSkipper( - indexCreatedVersion, - useDocValuesSkipper, - docValues.getValue(), - indexMode, - indexSortConfig, - fullFieldName - ); + boolean hasDocValuesSkipper = shouldUseDocValuesSkipper(indexSettings, docValues.getValue(), fullFieldName); if (hasDocValuesSkipper) { return IndexType.skippers(); } @@ -467,42 +440,17 @@ public DateFieldMapper build(MapperBuilderContext context) { nullTimestamp, resolution, context.isSourceSynthetic(), - indexMode, - indexSortConfig, - indexType.hasDocValuesSkipper(), this ); } } public static final TypeParser MILLIS_PARSER = createTypeParserWithLegacySupport((n, c) -> { - boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings()); - return new Builder( - n, - Resolution.MILLISECONDS, - c.getDateFormatter(), - c.scriptCompiler(), - ignoreMalformedByDefault, - c.getIndexSettings().getMode(), - c.getIndexSettings().getIndexSortConfig(), - c.indexVersionCreated(), - IndexSettings.USE_DOC_VALUES_SKIPPER.get(c.getSettings()) - ); + return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), c.scriptCompiler(), c.getIndexSettings()); }); public static final TypeParser NANOS_PARSER = createTypeParserWithLegacySupport((n, c) -> { - boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings()); - return new Builder( - n, - Resolution.NANOSECONDS, - c.getDateFormatter(), - c.scriptCompiler(), - ignoreMalformedByDefault, - c.getIndexSettings().getMode(), - c.getIndexSettings().getIndexSortConfig(), - c.indexVersionCreated(), - IndexSettings.USE_DOC_VALUES_SKIPPER.get(c.getSettings()) - ); + return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), c.scriptCompiler(), c.getIndexSettings()); }); public static final class DateFieldType extends MappedFieldType { @@ -1114,17 +1062,12 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final Resolution resolution; private final boolean isSourceSynthetic; - private final boolean ignoreMalformedByDefault; - private final IndexVersion indexCreatedVersion; - private final Script script; private final ScriptCompiler scriptCompiler; private final FieldValues scriptValues; private final boolean isDataStreamTimestampField; - private final IndexMode indexMode; - private final IndexSortConfig indexSortConfig; - private final boolean hasDocValuesSkipper; + private final IndexSettings indexSettings; private DateFieldMapper( String leafName, @@ -1133,9 +1076,6 @@ private DateFieldMapper( Long nullValue, Resolution resolution, boolean isSourceSynthetic, - IndexMode indexMode, - IndexSortConfig indexSortConfig, - boolean hasDocValuesSkipper, Builder builder ) { super(leafName, mappedFieldType, builderParams); @@ -1149,15 +1089,11 @@ private DateFieldMapper( this.nullValue = nullValue; this.resolution = resolution; this.isSourceSynthetic = isSourceSynthetic; - this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue(); - this.indexCreatedVersion = builder.indexCreatedVersion; this.script = builder.script.get(); this.scriptCompiler = builder.scriptCompiler; this.scriptValues = builder.scriptValues(); this.isDataStreamTimestampField = mappedFieldType.name().equals(DataStreamTimestampFieldMapper.DEFAULT_PATH); - this.indexMode = indexMode; - this.indexSortConfig = indexSortConfig; - this.hasDocValuesSkipper = hasDocValuesSkipper; + this.indexSettings = builder.indexSettings; } /** @@ -1168,46 +1104,25 @@ private DateFieldMapper( * field has doc values enabled. Additionally, the index mode must be {@link IndexMode#LOGSDB} or {@link IndexMode#TIME_SERIES}, and * the index sorting configuration must include the {@code @timestamp} field. * - * @param indexCreatedVersion The version of the index when it was created. - * @param useDocValuesSkipper Whether the doc values skipper feature is enabled via the {@code index.mapping.use_doc_values_skipper} - * setting. - * @param hasDocValues Whether the field has doc values enabled. - * @param indexMode The index mode, which must be {@link IndexMode#LOGSDB} or {@link IndexMode#TIME_SERIES}. - * @param indexSortConfig The index sorting configuration, which must include the {@code @timestamp} field. - * @param fullFieldName The full name of the field being checked, expected to be {@code @timestamp}. + * @param indexSettings The index settings of the parent index + * @param hasDocValues Whether the field has doc values enabled. + * @param fullFieldName The full name of the field being checked, expected to be {@code @timestamp}. * @return {@code true} if the doc values skipper should be used, {@code false} otherwise. */ - private static boolean shouldUseDocValuesSkipper( - final IndexVersion indexCreatedVersion, - boolean useDocValuesSkipper, - boolean hasDocValues, - final IndexMode indexMode, - final IndexSortConfig indexSortConfig, - final String fullFieldName - ) { - return indexCreatedVersion.onOrAfter(IndexVersions.REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX) - && useDocValuesSkipper + private static boolean shouldUseDocValuesSkipper(IndexSettings indexSettings, boolean hasDocValues, final String fullFieldName) { + return indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX) + && indexSettings.useDocValuesSkipper() && hasDocValues - && (IndexMode.LOGSDB.equals(indexMode) || IndexMode.TIME_SERIES.equals(indexMode)) - && indexSortConfig != null - && indexSortConfig.hasSortOnField(fullFieldName) + && (IndexMode.LOGSDB.equals(indexSettings.getMode()) || IndexMode.TIME_SERIES.equals(indexSettings.getMode())) + && indexSettings.getIndexSortConfig() != null + && indexSettings.getIndexSortConfig().hasSortOnField(fullFieldName) && DataStreamTimestampFieldMapper.DEFAULT_PATH.equals(fullFieldName); } @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder( - leafName(), - resolution, - null, - scriptCompiler, - ignoreMalformedByDefault, - indexMode, - indexSortConfig, - indexCreatedVersion, - hasDocValuesSkipper - ).init(this); + return new Builder(leafName(), resolution, null, scriptCompiler, indexSettings).init(this); } @Override @@ -1277,7 +1192,7 @@ private void indexValue(DocumentParserContext context, long timestamp) { DataStreamTimestampFieldMapper.storeTimestampValueForReuse(context.doc(), timestamp); } - if (hasDocValuesSkipper && hasDocValues) { + if (fieldType().hasDocValuesSkipper()) { context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), timestamp)); } else if (indexed && hasDocValues) { context.doc().add(new LongField(fieldType().name(), timestamp, Field.Store.NO)); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index e0d89afab2b24..d8647de21523e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.xcontent.XContentParser; @@ -412,11 +411,7 @@ public boolean newDynamicDateField(DocumentParserContext context, String name, D DateFieldMapper.Resolution.MILLISECONDS, dateTimeFormatter, ScriptCompiler.NONE, - ignoreMalformed, - context.indexSettings().getMode(), - context.indexSettings().getIndexSortConfig(), - context.indexSettings().getIndexVersionCreated(), - IndexSettings.USE_DOC_VALUES_SKIPPER.get(context.indexSettings().getSettings()) + context.indexSettings() ), context ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index db4022c8be539..6ded77738ec9b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -56,7 +56,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -756,19 +755,28 @@ public void testLegacyField() throws Exception { assertNotEquals(DEFAULT_DATE_TIME_FORMATTER, ((DateFieldType) service.fieldType("mydate")).dateTimeFormatter); } + private static IndexSettings buildIndexSettings(IndexVersion indexVersion) { + return new IndexSettings( + new IndexMetadata.Builder("index").settings(indexSettings(indexVersion, 1, 1).build()).build(), + Settings.EMPTY + ); + } + public void testLegacyDateFormatName() { + + // BWC compatible index, e.g 7.x + IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_7_0_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + ); + DateFieldMapper.Builder builder = new DateFieldMapper.Builder( "format", DateFieldMapper.Resolution.MILLISECONDS, null, mock(ScriptService.class), - true, - // BWC compatible index, e.g 7.x - IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_7_0_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) - ) + buildIndexSettings(indexVersion) ); // Check that we allow the use of camel case date formats on 7.x indices @@ -785,8 +793,7 @@ public void testLegacyDateFormatName() { DateFieldMapper.Resolution.MILLISECONDS, null, mock(ScriptService.class), - true, - IndexVersion.current() + buildIndexSettings(IndexVersion.current()) ); @SuppressWarnings("unchecked") diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index abaf363dfebe5..119553a6f180f 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -705,8 +705,7 @@ public static MetadataRolloverService getMetadataRolloverService( DateFieldMapper.Resolution.MILLISECONDS, null, ScriptCompiler.NONE, - false, - IndexVersion.current() + DEFAULT_INDEX_SETTINGS ).build(MapperBuilderContext.root(false, true)); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); @@ -723,9 +722,8 @@ public static MetadataRolloverService getMetadataRolloverService( DateFieldMapper.Resolution.MILLISECONDS, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, - true, - IndexVersion.current() - ) + DEFAULT_INDEX_SETTINGS + ).ignoreMalformed(true) ); MetadataFieldMapper dtfm = getDataStreamTimestampFieldMapper(); Mapping mapping = new Mapping( @@ -764,21 +762,21 @@ public static MetadataRolloverService getMetadataRolloverService( ); } + private static final IndexSettings DEFAULT_INDEX_SETTINGS = new IndexSettings( + IndexMetadata.builder("_na_") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(), + Settings.EMPTY + ); + public static MetadataFieldMapper getDataStreamTimestampFieldMapper() { Map fieldsMapping = new HashMap<>(); fieldsMapping.put("enabled", true); MappingParserContext mockedParserContext = mock(MappingParserContext.class); - when(mockedParserContext.getIndexSettings()).thenReturn( - new IndexSettings( - IndexMetadata.builder("_na_") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .creationDate(System.currentTimeMillis()) - .build(), - Settings.EMPTY - ) - ); + when(mockedParserContext.getIndexSettings()).thenReturn(DEFAULT_INDEX_SETTINGS); return DataStreamTimestampFieldMapper.PARSER.parse("field", fieldsMapping, mockedParserContext).build(); }