-
Notifications
You must be signed in to change notification settings - Fork 24.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Change deprecation indexing to use a custom template (#64417)
The implementation for indexing deprecation logs to a data stream (#58924) relied on the Stack template for `logs-*-*`. This meant that if the user disabled the stack templates, the templates would also be unavailable for the deprecation logs. Change the implementation so that: * There is a separate template for deprecation logging * The data stream is marked as hidden * The data stream name is prefixed with a period (`.`)
- Loading branch information
1 parent
1990e24
commit 39a8643
Showing
10 changed files
with
211 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 12 additions & 0 deletions
12
...c/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
{ | ||
"phases": { | ||
"hot": { | ||
"actions": { | ||
"rollover": { | ||
"max_size": "50gb", | ||
"max_age": "30d" | ||
} | ||
} | ||
} | ||
} | ||
} |
60 changes: 60 additions & 0 deletions
60
...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
{ | ||
"template": { | ||
"mappings": { | ||
"dynamic_templates": [ | ||
{ | ||
"strings_as_keyword": { | ||
"mapping": { | ||
"ignore_above": 1024, | ||
"type": "keyword" | ||
}, | ||
"match_mapping_type": "string" | ||
} | ||
} | ||
], | ||
"date_detection": false, | ||
"properties": { | ||
"@timestamp": { | ||
"type": "date" | ||
}, | ||
"data_stream": { | ||
"properties": { | ||
"type": { | ||
"type": "constant_keyword", | ||
"value": "logs" | ||
}, | ||
"dataset": { | ||
"type": "constant_keyword" | ||
}, | ||
"namespace": { | ||
"type": "constant_keyword" | ||
} | ||
} | ||
}, | ||
"ecs": { | ||
"properties": { | ||
"version": { | ||
"ignore_above": 1024, | ||
"type": "keyword" | ||
} | ||
} | ||
}, | ||
"host": { | ||
"properties": { | ||
"ip": { | ||
"type": "ip" | ||
} | ||
} | ||
}, | ||
"message": { | ||
"type": "text" | ||
} | ||
} | ||
} | ||
}, | ||
"_meta": { | ||
"description": "default mappings for ES deprecation logs index template installed by x-pack", | ||
"managed": true | ||
}, | ||
"version": ${xpack.deprecation.indexing.template.version} | ||
} |
20 changes: 20 additions & 0 deletions
20
...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
{ | ||
"template": { | ||
"settings": { | ||
"index": { | ||
"lifecycle": { | ||
"name": ".deprecation-indexing-ilm-policy" | ||
}, | ||
"codec": "best_compression", | ||
"query": { | ||
"default_field": ["message"] | ||
} | ||
} | ||
} | ||
}, | ||
"_meta": { | ||
"description": "default settings for ES deprecation logs index template installed by x-pack", | ||
"managed": true | ||
}, | ||
"version": ${xpack.deprecation.indexing.template.version} | ||
} |
17 changes: 17 additions & 0 deletions
17
...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
{ | ||
"index_patterns": [".logs-deprecation-elasticsearch"], | ||
"priority": 1000, | ||
"data_stream": { | ||
"hidden": true | ||
}, | ||
"composed_of": [ | ||
".deprecation-indexing-mappings", | ||
".deprecation-indexing-settings" | ||
], | ||
"allow_auto_create": true, | ||
"_meta": { | ||
"description": "default template for ES deprecation logs index template installed by x-pack", | ||
"managed": true | ||
}, | ||
"version": ${xpack.deprecation.indexing.template.version} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
92 changes: 92 additions & 0 deletions
92
...java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License; | ||
* you may not use this file except in compliance with the Elastic License. | ||
*/ | ||
|
||
package org.elasticsearch.xpack.deprecation.logging; | ||
|
||
import org.elasticsearch.client.Client; | ||
import org.elasticsearch.cluster.service.ClusterService; | ||
import org.elasticsearch.common.settings.Settings; | ||
import org.elasticsearch.common.xcontent.NamedXContentRegistry; | ||
import org.elasticsearch.threadpool.ThreadPool; | ||
import org.elasticsearch.xpack.core.template.IndexTemplateConfig; | ||
import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; | ||
import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; | ||
|
||
import java.util.List; | ||
|
||
import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN; | ||
|
||
/** | ||
* Manages the index template and associated ILM policy for deprecation log indexing. | ||
*/ | ||
public class DeprecationIndexingTemplateRegistry extends IndexTemplateRegistry { | ||
// history (please add a comment why you increased the version here) | ||
// version 1: initial | ||
public static final int INDEX_TEMPLATE_VERSION = 1; | ||
|
||
public static final String DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE = "xpack.deprecation.indexing.template.version"; | ||
|
||
public static final String DEPRECATION_INDEXING_MAPPINGS_NAME = ".deprecation-indexing-mappings"; | ||
public static final String DEPRECATION_INDEXING_SETTINGS_NAME = ".deprecation-indexing-settings"; | ||
public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template"; | ||
public static final String DEPRECATION_INDEXING_POLICY_NAME = ".deprecation-indexing-ilm-policy"; | ||
|
||
public static final IndexTemplateConfig DEPRECATION_INDEXING_MAPPINGS = new IndexTemplateConfig( | ||
DEPRECATION_INDEXING_MAPPINGS_NAME, | ||
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json", | ||
INDEX_TEMPLATE_VERSION, | ||
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE | ||
); | ||
|
||
public static final IndexTemplateConfig DEPRECATION_INDEXING_SETTINGS = new IndexTemplateConfig( | ||
DEPRECATION_INDEXING_SETTINGS_NAME, | ||
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json", | ||
INDEX_TEMPLATE_VERSION, | ||
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE | ||
); | ||
|
||
public static final IndexTemplateConfig DEPRECATION_INDEXING_INDEX_TEMPLATE = new IndexTemplateConfig( | ||
DEPRECATION_INDEXING_TEMPLATE_NAME, | ||
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json", | ||
INDEX_TEMPLATE_VERSION, | ||
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE | ||
); | ||
|
||
public static final LifecyclePolicyConfig DEPRECATION_INDEXING_HISTORY_POLICY = new LifecyclePolicyConfig( | ||
DEPRECATION_INDEXING_POLICY_NAME, | ||
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json" | ||
); | ||
|
||
public DeprecationIndexingTemplateRegistry( | ||
Settings nodeSettings, | ||
ClusterService clusterService, | ||
ThreadPool threadPool, | ||
Client client, | ||
NamedXContentRegistry xContentRegistry | ||
) { | ||
super(nodeSettings, clusterService, threadPool, client, xContentRegistry); | ||
} | ||
|
||
@Override | ||
protected List<IndexTemplateConfig> getComponentTemplateConfigs() { | ||
return List.of(DEPRECATION_INDEXING_MAPPINGS, DEPRECATION_INDEXING_SETTINGS); | ||
} | ||
|
||
@Override | ||
protected List<IndexTemplateConfig> getComposableTemplateConfigs() { | ||
return List.of(DEPRECATION_INDEXING_INDEX_TEMPLATE); | ||
} | ||
|
||
@Override | ||
protected List<LifecyclePolicyConfig> getPolicyConfigs() { | ||
return List.of(DEPRECATION_INDEXING_HISTORY_POLICY); | ||
} | ||
|
||
@Override | ||
protected String getOrigin() { | ||
return DEPRECATION_ORIGIN; | ||
} | ||
} |