Skip to content

Commit

Permalink
Introduce experimental pass-through field type (#103648)
Browse files Browse the repository at this point in the history 
* Introduce passthrough field type

`PassthoughObjectMapper` extends `ObjectMapper` to create a container
for fields that also need to be referenced as if they were at the root
level. This is done by creating aliases for all its subfields.

It also supports an option of annotating all its subfields as
dimensions. This will be leveraged in TSDB, where dimension fields can
be dynamically defined as nested under a passthrough object - and still
referenced directly (i.e. without prefixes) in aggregation queries.

Related to #103567

* Update docs/changelog/103648.yaml

* no subobjects

* create dimensions dynamically

* remove unused method

* restore ignoreAbove incompatibility with dimension

* fix test

* refactor, skip aliases on conflict

* fix branch

* fix branch

* add tests

* update test

* remove unused variable

* add yaml test for subobject

* minor refactoring

* add unittest for PassThroughObjectMapper

* suggested fixes

* suggested fixes

* update yaml with warning for duplicate alias

* updates from review

* add withoutMappers()
  • Loading branch information
@kkrik-es
kkrik-es committed Feb 1, 2024
1 parent bdd3a4f commit 149ec37
Show file tree
Hide file tree
Showing 25 changed files with 872 additions and 27 deletions.
5 changes: 5 additions & 0 deletions docs/changelog/103648.yaml
View file
@@ -0,0 +1,5 @@
pr: 103648
summary: Introduce experimental pass-through field type
area: TSDB
type: enhancement
issues: []
11 changes: 10 additions & 1 deletion ...-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java
View file
Expand Up @@ -26,6 +26,7 @@
import org.elasticsearch.index.mapper.MapperBuilderContext;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MappingParserContext;
import org.elasticsearch.index.mapper.PassThroughObjectMapper;

import java.io.IOException;
import java.io.UncheckedIOException;
Expand Down Expand Up @@ -152,8 +153,9 @@ private List<String> findRoutingPaths(String indexName, Settings allSettings, Li
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
// Avoid failing because index.routing_path is missing
.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD)
.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"))
.build();

tmpIndexMetadata.settings(finalResolvedSettings);
Expand All @@ -164,6 +166,13 @@ private List<String> findRoutingPaths(String indexName, Settings allSettings, Li
for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) {
extractPath(routingPaths, fieldMapper);
}
for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) {
if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) {
if (passThroughObjectMapper.containsDimensions()) {
routingPaths.add(passThroughObjectMapper.fullPath() + ".*");
}
}
}
for (var template : mapperService.getAllDynamicTemplates()) {
if (template.pathMatch().isEmpty()) {
continue;
Expand Down
27 changes: 27 additions & 0 deletions ...ams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java
View file
Expand Up @@ -638,6 +638,33 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro
assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size());
}

public void testGenerateRoutingPathFromPassThroughObject() throws Exception {
Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
String mapping = """
{
"_doc": {
"properties": {
"labels": {
"type": "passthrough",
"time_series_dimension": true
},
"metrics": {
"type": "passthrough"
},
"another_field": {
"type": "keyword"
}
}
}
}
""";
Settings result = generateTsdbSettings(mapping, now);
assertThat(result.size(), equalTo(3));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*"));
}

private Settings generateTsdbSettings(String mapping, Instant now) throws IOException {
Metadata metadata = Metadata.EMPTY_METADATA;
String dataStreamName = "logs-app1";
Expand Down
259 changes: 259 additions & 0 deletions modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
View file
Expand Up @@ -191,3 +191,262 @@ index without timestamp with pipeline:
pipeline: my_pipeline
body:
- '{"@timestamp": "wrong_format", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'

---
dynamic templates:
- skip:
version: " - 8.12.99"
reason: "Support for dynamic fields was added in 8.13"
- do:
indices.put_index_template:
name: my-dynamic-template
body:
index_patterns: [k9s*]
data_stream: {}
template:
settings:
index:
number_of_shards: 1
mode: time_series
time_series:
start_time: 2023-08-31T13:03:08.138Z

mappings:
properties:
attributes:
type: passthrough
dynamic: true
time_series_dimension: true
dynamic_templates:
- counter_metric:
mapping:
type: integer
time_series_metric: counter

- do:
bulk:
index: k9s
refresh: true
body:
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "attributes.another.dim": "C" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "attributes.another.dim": "C" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "attributes.another.dim": "D" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "attributes.another.dim": "D" }'

- do:
search:
index: k9s
body:
size: 0

- match: { hits.total.value: 4 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
dim: A
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: { "attributes.another.dim": "C", "attributes.dim": "A" } }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
another.dim: C
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: { "attributes.another.dim": "C", "attributes.dim": "A" } }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

---
dynamic templates - conflicting aliases:
- skip:
version: " - 8.12.99"
reason: "Support for dynamic fields was added in 8.13"
- do:
indices.put_index_template:
name: my-dynamic-template
body:
index_patterns: [k9s*]
data_stream: {}
template:
settings:
index:
number_of_shards: 1
mode: time_series
time_series:
start_time: 2023-08-31T13:03:08.138Z

mappings:
properties:
attributes:
type: passthrough
dynamic: true
time_series_dimension: true
resource_attributes:
type: passthrough
dynamic: true
time_series_dimension: true
dynamic_templates:
- counter_metric:
mapping:
type: integer
time_series_metric: counter

- do:
bulk:
index: k9s
refresh: true
body:
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "resource_attributes.dim": "C" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "resource_attributes.dim": "C" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "resource_attributes.dim": "D" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "resource_attributes.dim": "D" }'

- do:
search:
index: k9s
body:
size: 0

- match: { hits.total.value: 4 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
dim: "C"
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: { "resource_attributes.dim": "C", "attributes.dim": "A" } }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
attributes.dim: A
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: { "resource_attributes.dim": "C", "attributes.dim": "A" } }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

---
dynamic templates - subobject in passthrough object error:
- skip:
version: " - 8.12.99"
reason: "Support for dynamic fields was added in 8.13"
- do:
catch: /Tried to add subobject \[subcategory\] to object \[attributes\] which does not support subobjects/
indices.put_index_template:
name: my-dynamic-template
body:
index_patterns: [k9s*]
data_stream: {}
template:
settings:
index:
mode: time_series

mappings:
properties:
attributes:
type: passthrough
properties:
subcategory:
type: object
properties:
dim:
type: keyword

- do:
catch: /Mapping definition for \[attributes\] has unsupported parameters:\ \[subobjects \:\ true\]/
indices.put_index_template:
name: my-dynamic-template
body:
index_patterns: [k9s*]
data_stream: {}
template:
settings:
index:
number_of_shards: 1
mode: time_series
time_series:
start_time: 2023-08-31T13:03:08.138Z

mappings:
properties:
attributes:
type: passthrough
subobjects: true

---
dynamic templates - passthrough not under root error:
- skip:
version: " - 8.12.99"
reason: "Support for dynamic fields was added in 8.13"
- do:
catch: /Tried to add passthrough subobject \[attributes\] to object \[resource\], passthrough is not supported as a subobject/
indices.put_index_template:
name: my-dynamic-template
body:
index_patterns: [k9s*]
data_stream: {}
template:
settings:
index:
mode: time_series

mappings:
properties:
"resource.attributes":
type: passthrough
dynamic: true
time_series_dimension: true
4 changes: 4 additions & 0 deletions server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
View file
Expand Up @@ -248,6 +248,10 @@ public static class ExtractFromSource extends IndexRouting {
this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true);
}

public boolean matchesField(String fieldName) {
return isRoutingPath.test(fieldName);
}

@Override
public void process(IndexRequest indexRequest) {}

Expand Down
7 changes: 6 additions & 1 deletion server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
View file
Expand Up @@ -552,7 +552,12 @@ public final MapperBuilderContext createDynamicMapperBuilderContext() {
if (p.endsWith(".")) {
p = p.substring(0, p.length() - 1);
}
return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false);
boolean containsDimensions = false;
ObjectMapper objectMapper = mappingLookup.objectMappers().get(p);
if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) {
containsDimensions = passThroughObjectMapper.containsDimensions();
}
return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false, containsDimensions);
}

public abstract XContentParser parser();
Expand Down

0 comments on commit 149ec37

Please sign in to comment.