Skip to content

Inject the host.name field mapping only if required for logsdb index mode #114573

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Oct 16, 2024
Merged

Inject the host.name field mapping only if required for logsdb index mode #114573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
be2c6b1
feature: inject the host.name field only if required for sorting
salvatore-campagna Oct 10, 2024
e3083bf
Merge branch 'main' into feature/inject-host-name-if-required
salvatore-campagna Oct 11, 2024
192da05
refactor: static mapping cretion for logsdb and time_series
salvatore-campagna Oct 11, 2024
42cc978
Merge branch 'main' into feature/inject-host-name-if-required
elasticmachine Oct 11, 2024
176cae4
Merge branch 'main' into feature/inject-host-name-if-required
elasticmachine Oct 11, 2024
a267dbd
Merge branch 'main' into feature/inject-host-name-if-required
elasticmachine Oct 11, 2024
6c76b56
Merge branch 'main' into feature/inject-host-name-if-required
elasticmachine Oct 11, 2024
14c1736
Merge branch 'main' into feature/inject-host-name-if-required
elasticmachine Oct 14, 2024
59d0837
fix: remove duplicate test
salvatore-campagna Oct 14, 2024
5cd2dac
test: add a few more tests
salvatore-campagna Oct 14, 2024
bb585ea
test: text fields do not have doc values
salvatore-campagna Oct 14, 2024
046f208
refactor: getDefault method and add some more tests
salvatore-campagna Oct 16, 2024
1ca0351
nit: rename createMapping to createDefaultMapping
salvatore-campagna Oct 16, 2024
60b9b8a
Merge branch 'main' into feature/inject-host-name-if-required
elasticmachine Oct 16, 2024
6f897a8
Merge branch 'main' into feature/inject-host-name-if-required
elasticmachine Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1373,7 +1373,7 @@ private static void updateIndexMappingsAndBuildSortOrder(
MapperService mapperService = indexService.mapperService();
IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD;
List<CompressedXContent> allMappings = new ArrayList<>();
final CompressedXContent defaultMapping = indexMode.getDefaultMapping();
final CompressedXContent defaultMapping = indexMode.getDefaultMapping(indexService.getIndexSettings());
if (defaultMapping != null) {
allMappings.add(defaultMapping);
}
Expand Down
71 changes: 30 additions & 41 deletions server/src/main/java/org/elasticsearch/index/IndexMode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
}

@Override
public CompressedXContent getDefaultMapping() {
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
return null;
}

Expand Down Expand Up @@ -171,7 +171,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
}

@Override
public CompressedXContent getDefaultMapping() {
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
}

Expand Down Expand Up @@ -249,8 +249,10 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
}

@Override
public CompressedXContent getDefaultMapping() {
return DEFAULT_LOGS_TIMESTAMP_MAPPING;
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
return indexSettings != null && indexSettings.getIndexSortConfig().hasPrimarySortOnField(HOST_NAME)
? DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME
: DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
}

@Override
Expand Down Expand Up @@ -308,6 +310,8 @@ public String getDefaultCodec() {
}
};

private static final String HOST_NAME = "host.name";

private static void validateTimeSeriesSettings(Map<Setting<?>, Object> settings) {
settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
Expand All @@ -324,48 +328,33 @@ protected static String tsdbMode() {
return "[" + IndexSettings.MODE.getKey() + "=time_series]";
}

public static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
private static CompressedXContent createDefaultMapping(boolean includeHostName) throws IOException {
return new CompressedXContent((builder, params) -> {
builder.startObject(MapperService.SINGLE_MAPPING_NAME)
.startObject(DataStreamTimestampFieldMapper.NAME)
.field("enabled", true)
.endObject()
.startObject("properties")
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
.field("type", DateFieldMapper.CONTENT_TYPE)
.endObject();

if (includeHostName) {
builder.startObject(HOST_NAME).field("type", KeywordFieldMapper.CONTENT_TYPE).field("ignore_above", 1024).endObject();
}

static {
try {
DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = new CompressedXContent(
((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
.startObject(DataStreamTimestampFieldMapper.NAME)
.field("enabled", true)
.endObject()
.startObject("properties")
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
.field("type", DateFieldMapper.CONTENT_TYPE)
.field("ignore_malformed", "false")
.endObject()
.endObject()
.endObject())
);
} catch (IOException e) {
throw new AssertionError(e);
}
return builder.endObject().endObject();
});
}

public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING;
private static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;

private static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME;

static {
try {
DEFAULT_LOGS_TIMESTAMP_MAPPING = new CompressedXContent(
((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
.startObject(DataStreamTimestampFieldMapper.NAME)
.field("enabled", true)
.endObject()
.startObject("properties")
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
.field("type", DateFieldMapper.CONTENT_TYPE)
.endObject()
.startObject("host.name")
.field("type", KeywordFieldMapper.CONTENT_TYPE)
.field("ignore_above", 1024)
.endObject()
.endObject()
.endObject())
);
DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = createDefaultMapping(false);
DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME = createDefaultMapping(true);
} catch (IOException e) {
throw new AssertionError(e);
}
Expand Down Expand Up @@ -421,7 +410,7 @@ public String getName() {
* Get default mapping for this index or {@code null} if there is none.
*/
@Nullable
public abstract CompressedXContent getDefaultMapping();
public abstract CompressedXContent getDefaultMapping(IndexSettings indexSettings);

/**
* Build the {@link FieldMapper} for {@code _id}.
Expand Down
16 changes: 14 additions & 2 deletions server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,24 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;

public class LogsIndexModeTests extends ESTestCase {
public void testLogsIndexModeSetting() {
assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGSDB));
}

public void testSortField() {
public void testDefaultHostNameSortField() {
final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true));
assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), containsString("host.name"));
}

public void testCustomSortField() {
final Settings sortSettings = Settings.builder()
.put(buildSettings())
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
Expand All @@ -29,7 +39,9 @@ public void testSortField() {
assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB));
assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
assertThat(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()), equalTo("agent_id"));
assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(false));
assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), not(containsString("host")));
}

public void testSortMode() {
Expand Down
8 changes: 6 additions & 2 deletions test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) {}
mapperMetrics
);

if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping() != null) {
mapperService.merge(null, indexSettings.getMode().getDefaultMapping(), MapperService.MergeReason.MAPPING_UPDATE);
if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping(indexSettings) != null) {
mapperService.merge(
null,
indexSettings.getMode().getDefaultMapping(indexSettings),
MapperService.MergeReason.MAPPING_UPDATE
);
}

return mapperService;
Expand Down
Loading