Skip to content

Commit

Permalink
Enforce synthetic source for time series indices (#93380)
Browse files Browse the repository at this point in the history 
Support for synthetic source is also added to `unsigned_long` field as part of this change.
This is required because `unsigned_long` field types can be used in tsdb indices and
this change would prohibit the usage of these field type otherwise.

Closes #92319
  • Loading branch information
@martijnvg
martijnvg committed Feb 2, 2023
1 parent 190b617 commit 9babcc9
Show file tree
Hide file tree
Showing 13 changed files with 349 additions and 143 deletions.
121 changes: 121 additions & 0 deletions rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -413,3 +413,124 @@ nested fields:
rx:
type: long
time_series_metric: gauge

---
regular source:
- skip:
version: " - 8.6.99"
reason: synthetic source

- do:
catch: '/time series indices only support synthetic source/'
indices.create:
index: tsdb_index
body:
settings:
index:
mode: time_series
routing_path: [k8s.pod.uid]
time_series:
start_time: 2021-04-28T00:00:00Z
end_time: 2021-04-29T00:00:00Z
mappings:
_source:
mode: stored
properties:
"@timestamp":
type: date
k8s:
properties:
pod:
properties:
uid:
type: keyword
time_series_dimension: true
---
disabled source:
- skip:
version: " - 8.6.99"
reason: synthetic source

- do:
catch: '/time series indices only support synthetic source/'
indices.create:
index: tsdb_index
body:
settings:
index:
mode: time_series
routing_path: [k8s.pod.uid]
time_series:
start_time: 2021-04-28T00:00:00Z
end_time: 2021-04-29T00:00:00Z
mappings:
_source:
mode: disabled
properties:
"@timestamp":
type: date
k8s:
properties:
pod:
properties:
uid:
type: keyword
time_series_dimension: true

---
source include/exclude:
- skip:
version: " - 8.6.99"
reason: synthetic source

- do:
catch: '/filtering the stored _source is incompatible with synthetic source/'
indices.create:
index: tsdb_index
body:
settings:
index:
mode: time_series
routing_path: [k8s.pod.uid]
time_series:
start_time: 2021-04-28T00:00:00Z
end_time: 2021-04-29T00:00:00Z
mappings:
_source:
includes: [a]
properties:
"@timestamp":
type: date
k8s:
properties:
pod:
properties:
uid:
type: keyword
time_series_dimension: true

- do:
catch: '/filtering the stored _source is incompatible with synthetic source/'
indices.create:
index: tsdb_index
body:
settings:
index:
mode: time_series
routing_path: [k8s.pod.uid]
time_series:
start_time: 2021-04-28T00:00:00Z
end_time: 2021-04-29T00:00:00Z
mappings:
_source:
excludes: [b]
properties:
"@timestamp":
type: date
k8s:
properties:
pod:
properties:
uid:
type: keyword
time_series_dimension: true
91 changes: 0 additions & 91 deletions rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,94 +148,3 @@ clone:

- match: {hits.total.value: 1}
- match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}

---
clone no source index:
- skip:
version: " - 8.1.99"
reason: tsdb indexing changed in 8.2.0

- do:
indices.create:
index: test_no_source
body:
settings:
index:
mode: time_series
routing_path: [ metricset, k8s.pod.uid ]
time_series:
start_time: 2021-04-28T00:00:00Z
end_time: 2021-04-29T00:00:00Z
number_of_shards: 1
number_of_replicas: 0
mappings:
_source:
enabled: false
properties:
"@timestamp":
type: date
metricset:
type: keyword
time_series_dimension: true
k8s:
properties:
pod:
properties:
uid:
type: keyword
time_series_dimension: true
name:
type: keyword
ip:
type: ip
network:
properties:
tx:
type: long
rx:
type: long

- do:
bulk:
refresh: true
index: test_no_source
body:
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'

- do:
indices.put_settings:
index: test_no_source
body:
index.blocks.write: true

- do:
indices.clone:
index: test_no_source
target: test_no_source_clone

- do:
search:
index: test_no_source_clone
body:
docvalue_fields:
- field: _tsid
query:
query_string:
query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
- match: {hits.total.value: 1}
- match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}
11 changes: 10 additions & 1 deletion .../src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.fieldcaps.FieldCapabilities;
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
Expand Down Expand Up @@ -37,6 +38,7 @@
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MetadataFieldMapper;
import org.elasticsearch.index.mapper.SourceLoader;
import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader;
import org.elasticsearch.index.mapper.TimeSeriesParams;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
Expand Down Expand Up @@ -118,6 +120,7 @@ public void setUp() throws Exception {
.endObject()
.startObject("playlist")
.field("type", "text")
.field("store", true)
.endObject()
.startObject("some_dimension")
.field("type", "keyword")
Expand Down Expand Up @@ -845,7 +848,13 @@ protected String contentType() {

@Override
public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
throw new UnsupportedOperationException();
return new StringStoredFieldFieldLoader(name(), simpleName(), null) {
@Override
protected void write(XContentBuilder b, Object value) throws IOException {
BytesRef ref = (BytesRef) value;
b.utf8Value(ref.bytes, ref.offset, ref.length);
}
};
}

private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper());
Expand Down
16 changes: 16 additions & 0 deletions server/src/main/java/org/elasticsearch/index/IndexMode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.elasticsearch.index.mapper.NestedLookup;
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
import org.elasticsearch.index.mapper.RoutingFieldMapper;
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper;

Expand Down Expand Up @@ -110,6 +111,9 @@ public DocumentDimensions buildDocumentDimensions(IndexSettings settings) {
public boolean shouldValidateTimestamp() {
return false;
}

@Override
public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {}
},
TIME_SERIES("time_series") {
@Override
Expand Down Expand Up @@ -196,6 +200,13 @@ public DocumentDimensions buildDocumentDimensions(IndexSettings settings) {
public boolean shouldValidateTimestamp() {
return true;
}

@Override
public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
if (sourceFieldMapper.isSynthetic() == false) {
throw new IllegalArgumentException("time series indices only support synthetic source");
}
}
};

protected static String tsdbMode() {
Expand Down Expand Up @@ -310,6 +321,11 @@ public String getName() {
*/
public abstract boolean shouldValidateTimestamp();

/**
* Validates the source field mapper
*/
public abstract void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper);

/**
* Parse a string into an {@link IndexMode}.
*/
Expand Down
3 changes: 1 addition & 2 deletions server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,12 @@ public void validate(IndexSettings settings, boolean checkLimits) {
}
}

settings.getMode().validateMapping(mappingLookup);
/*
* Build an empty source loader to validate that the mapping is compatible
* with the source loading strategy declared on the source field mapper.
*/
sourceMapper().newSourceLoader(mapping());

settings.getMode().validateMapping(mappingLookup);
if (settings.getIndexSortConfig().hasIndexSort() && mappers().nestedLookup() != NestedLookup.EMPTY) {
throw new IllegalArgumentException("cannot have nested fields when index sort is activated");
}
Expand Down

0 comments on commit 9babcc9

Please sign in to comment.