Skip to content

Commit

Permalink
[8.13] Handle pass-through subfields with deep nesting (#106798)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kkrik-es
kkrik-es committed Mar 27, 2024
1 parent ad39c3f commit 10d088d
Show file tree
Hide file tree
Showing 3 changed files with 373 additions and 16 deletions.
231 changes: 222 additions & 9 deletions modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
View file
Expand Up @@ -463,13 +463,13 @@ dynamic templates with nesting:
refresh: true
body:
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10" }'
- '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.a.much.deeper.nested.dim": "AC" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10" }'
- '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.a.much.deeper.nested.dim": "AC" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20" }'
- '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.a.much.deeper.nested.dim": "BD" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20" }'
- '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.a.much.deeper.nested.dim": "BD" }'

- do:
search:
Expand All @@ -495,7 +495,7 @@ dynamic templates with nesting:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
Expand All @@ -514,7 +514,7 @@ dynamic templates with nesting:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
Expand All @@ -533,7 +533,7 @@ dynamic templates with nesting:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
Expand All @@ -545,14 +545,227 @@ dynamic templates with nesting:
filterA:
filter:
term:
another.dim2: 10
another.dim2: 10.5
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
a.much.deeper.nested.dim: AC
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

---
dynamic templates with incremental indexing:
- skip:
version: " - 8.12.99"
reason: "Support for dynamic fields was added in 8.13"
- do:
allowed_warnings:
- "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
indices.put_index_template:
name: my-dynamic-template
body:
index_patterns: [k9s*]
data_stream: {}
template:
settings:
index:
number_of_shards: 1
mode: time_series
time_series:
start_time: 2023-08-31T13:03:08.138Z

mappings:
properties:
attributes:
type: passthrough
dynamic: true
time_series_dimension: true
resource:
type: object
properties:
attributes:
type: passthrough
dynamic: true
time_series_dimension: true
dynamic_templates:
- counter_metric:
mapping:
type: integer
time_series_metric: counter

- do:
bulk:
index: k9s
refresh: true
body:
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "attributes.dim2": "C" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "attributes.dim2": "C" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "attributes.dim2": "D" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "attributes.dim2": "D" }'

- do:
bulk:
index: k9s
refresh: true
body:
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:04:08.138Z","data": "110", "resource.attributes.another.dim1": "1", "attributes.another.dim2": "10.5" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:04:09.138Z","data": "120", "resource.attributes.another.dim1": "1", "attributes.another.dim2": "10.5" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:04:10.138Z","data": "130", "resource.attributes.another.dim1": "2", "attributes.another.dim2": "20.5" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:04:10.238Z","data": "140", "resource.attributes.another.dim1": "2", "attributes.another.dim2": "20.5" }'

- do:
bulk:
index: k9s
refresh: true
body:
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:05:08.138Z","data": "210", "resource.attributes.another.deeper.dim1": "1", "attributes.another.deeper.dim2": "10.5" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:05:09.138Z","data": "220", "resource.attributes.another.deeper.dim1": "1", "attributes.another.deeper.dim2": "10.5" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:05:10.138Z","data": "230", "resource.attributes.another.deeper.dim1": "2", "attributes.another.deeper.dim2": "20.5" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:05:10.238Z","data": "240", "resource.attributes.another.deeper.dim1": "2", "attributes.another.deeper.dim2": "20.5" }'

- do:
bulk:
index: k9s
refresh: true
body:
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:06:08.138Z","data": "310", "attributes.a.much.deeper.nested.dim": "AC" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:06:09.138Z","data": "320", "attributes.a.much.deeper.nested.dim": "AC" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:06:10.138Z","data": "330", "attributes.a.much.deeper.nested.dim": "BD" }'
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
- '{ "@timestamp": "2023-09-01T13:06:10.238Z","data": "340", "attributes.a.much.deeper.nested.dim": "BD" }'

- do:
search:
index: k9s
body:
size: 0

- match: { hits.total.value: 16 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
dim1: A
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
dim2: C
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
another.deeper.dim1: 1
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
another.deeper.dim2: 10.5
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

- do:
search:
index: k9s
body:
size: 0
aggs:
filterA:
filter:
term:
a.much.deeper.nested.dim: AC
aggs:
tsids:
terms:
field: _tsid

- length: { aggregations.filterA.tsids.buckets: 1 }
- match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }

---
Expand Down
61 changes: 54 additions & 7 deletions server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
View file
Expand Up @@ -128,19 +128,22 @@ public RootObjectMapper build(MapperBuilderContext context) {
}

Map<String, Mapper> getAliasMappers(Map<String, Mapper> mappers, MapperBuilderContext context) {
Map<String, Mapper> aliasMappers = new HashMap<>();
Map<String, Mapper> newMappers = new HashMap<>();
Map<String, ObjectMapper.Builder> objectIntermediates = new HashMap<>(1);
getAliasMappers(mappers, aliasMappers, objectIntermediates, context, 0);
Map<String, ObjectMapper.Builder> objectIntermediatesFullName = new HashMap<>(1);
getAliasMappers(mappers, mappers, newMappers, objectIntermediates, objectIntermediatesFullName, context, 0);
for (var entry : objectIntermediates.entrySet()) {
aliasMappers.put(entry.getKey(), entry.getValue().build(context));
newMappers.put(entry.getKey(), entry.getValue().build(context));
}
return aliasMappers;
return newMappers;
}

void getAliasMappers(
Map<String, Mapper> mappers,
Map<String, Mapper> topLevelMappers,
Map<String, Mapper> aliasMappers,
Map<String, ObjectMapper.Builder> objectIntermediates,
Map<String, ObjectMapper.Builder> objectIntermediatesFullName,
MapperBuilderContext context,
int level
) {
Expand Down Expand Up @@ -179,32 +182,76 @@ void getAliasMappers(
).build(context);
aliasMappers.put(aliasMapper.simpleName(), aliasMapper);
} else {
conflict = topLevelMappers.get(fieldNameParts[0]);
if (conflict != null) {
if (isConflictingObject(conflict, fieldNameParts)) {
throw new IllegalArgumentException(
"Conflicting objects created during alias generation for pass-through field: ["
+ conflict.name()
+ "]"
);
}
}

// Nest the alias within object(s).
String realFieldName = fieldNameParts[fieldNameParts.length - 1];
Mapper.Builder fieldBuilder = new FieldAliasMapper.Builder(realFieldName).path(
fieldMapper.mappedFieldType.name()
);
ObjectMapper.Builder intermediate = null;
for (int i = fieldNameParts.length - 2; i >= 0; --i) {
String intermediateObjectName = fieldNameParts[i];
ObjectMapper.Builder intermediate = objectIntermediates.computeIfAbsent(
intermediateObjectName,
intermediate = objectIntermediatesFullName.computeIfAbsent(
concatStrings(fieldNameParts, i),
s -> new ObjectMapper.Builder(intermediateObjectName, ObjectMapper.Defaults.SUBOBJECTS)
);
intermediate.add(fieldBuilder);
fieldBuilder = intermediate;
}
objectIntermediates.putIfAbsent(fieldNameParts[0], intermediate);
}
}
}
}
} else if (mapper instanceof ObjectMapper objectMapper) {
// Call recursively to check child fields. The level guards against long recursive call sequences.
getAliasMappers(objectMapper.mappers, aliasMappers, objectIntermediates, context, level + 1);
getAliasMappers(
objectMapper.mappers,
topLevelMappers,
aliasMappers,
objectIntermediates,
objectIntermediatesFullName,
context,
level + 1
);
}
}
}
}

private static String concatStrings(String[] parts, int last) {
StringBuilder builder = new StringBuilder();
for (int i = 0; i <= last; i++) {
builder.append('.');
builder.append(parts[i]);
}
return builder.toString();
}

private static boolean isConflictingObject(Mapper mapper, String[] parts) {
for (int i = 0; i < parts.length - 1; i++) {
if (mapper == null) {
return true;
}
if (mapper instanceof ObjectMapper objectMapper) {
mapper = objectMapper.getMapper(parts[i + 1]);
} else {
return true;
}
}
return mapper == null;
}

private final Explicit<DateFormatter[]> dynamicDateTimeFormatters;
private final Explicit<Boolean> dateDetection;
private final Explicit<Boolean> numericDetection;
Expand Down

0 comments on commit 10d088d

Please sign in to comment.