From 08a19803b6833219439254be205b1eea7b8141df Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 5 Nov 2024 19:11:50 +0800 Subject: [PATCH 1/3] [apm-data] Apply lazy rollover on index template creation (#116219) * Apply lazy rollover on index template creation We should trigger a lazy rollover of existing data streams regardless of whether the index template is being created or updated. This ensures that the apm-data plugin will roll over data streams that were previously using the Fleet integration package. * Update docs/changelog/116219.yaml * Update docs/changelog/116219.yaml * Add YAML REST test for template reinstallation * Code review suggestion https://github.com/elastic/elasticsearch/pull/116219#discussion_r1828992554 * Remove wait_for_events from setup This doesn't guarantee the templates are set up, it only increases the chances; and we disable the plugin at the start of the test anyway. (cherry picked from commit b7b1ca041e2d96fe17d9baeaf80d6a575f61082a) # Conflicts: # x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java --- docs/changelog/116219.yaml | 6 + .../resources/rest-api-spec/test/10_apm.yml | 17 ++ .../rest-api-spec/test/10_rollover.yml | 51 ++++ .../RolloverEnabledTestTemplateRegistry.java | 2 +- .../core/template/IndexTemplateRegistry.java | 110 +++++---- .../core/template/YamlTemplateRegistry.java | 233 ++++++++++++++++++ .../template/IndexTemplateRegistryTests.java | 7 +- .../TestRegistryWithCustomPlugin.java | 2 +- 8 files changed, 378 insertions(+), 50 deletions(-) create mode 100644 docs/changelog/116219.yaml create mode 100644 x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_rollover.yml create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java diff --git a/docs/changelog/116219.yaml b/docs/changelog/116219.yaml new file mode 100644 index 0000000000000..aeeea68570e77 --- /dev/null +++ b/docs/changelog/116219.yaml @@ -0,0 +1,6 @@ +pr: 116219 +summary: "[apm-data] Apply lazy rollover on index template creation" +area: Data streams +type: bug +issues: + - 116230 diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml index 4b45fda66835c..101292c040df4 100644 --- a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml @@ -53,6 +53,23 @@ setup: - contains: {index_templates: {name: logs-apm.app@template}} - contains: {index_templates: {name: logs-apm.error@template}} +--- +"Test template reinstallation": + - skip: + reason: contains is a newly added assertion + features: contains + - do: + indices.delete_index_template: + name: traces-apm@template + - do: + cluster.health: + wait_for_events: languid + - do: + indices.get_index_template: + name: traces-apm@template + - length: {index_templates: 1} + - contains: {index_templates: {name: traces-apm@template}} + --- "Test traces-apm-* data stream indexing": - skip: diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_rollover.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_rollover.yml new file mode 100644 index 0000000000000..95586cd1fd665 --- /dev/null +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_rollover.yml @@ -0,0 +1,51 @@ +--- +setup: + - do: + indices.put_index_template: + name: traces-low-prio + body: + data_stream: {} + index_patterns: ["traces-*"] + priority: 1 + +--- +"Test data stream rollover on template installation": + - skip: + awaits_fix: "https://github.com/elastic/elasticsearch/issues/102360" + + # Disable the apm-data plugin and delete the traces-apm@template index + # template so traces-low-prio takes effect. + - do: + cluster.put_settings: + body: + transient: + xpack.apm_data.registry.enabled: false + - do: + indices.delete_index_template: + name: traces-apm@template + - do: + indices.create_data_stream: + name: traces-apm-testing + - do: + indices.get_data_stream: + name: traces-apm-testing + - match: {data_streams.0.template: traces-low-prio} + + # Re-enable the apm-data plugin, after which the traces-apm@template + # index template should be recreated and trigger a lazy rollover on + # the traces-apm-testing data stream. + - do: + cluster.put_settings: + body: + transient: + xpack.apm_data.registry.enabled: true + - do: + cluster.health: + wait_for_events: languid + - do: + indices.get_data_stream: + name: traces-apm-testing + - length: {data_streams: 1} + - match: {data_streams.0.template: traces-apm@template} + - match: {data_streams.0.rollover_on_write: true} + diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/RolloverEnabledTestTemplateRegistry.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/RolloverEnabledTestTemplateRegistry.java index 819b0e01ac4de..442ad9a68dfc4 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/RolloverEnabledTestTemplateRegistry.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/RolloverEnabledTestTemplateRegistry.java @@ -53,7 +53,7 @@ protected Map getComposableTemplateConfigs() { } @Override - protected boolean applyRolloverAfterTemplateV2Upgrade() { + protected boolean applyRolloverAfterTemplateV2Update() { return true; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index 87092c45bf032..aa44879a517d8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -400,7 +400,7 @@ private void addComposableTemplatesIfMissing(ClusterState state) { } } else if (Objects.isNull(currentTemplate)) { logger.debug("adding composable template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); - putComposableTemplate(state, templateName, newTemplate.getValue(), creationCheck, false); + putComposableTemplate(state, templateName, newTemplate.getValue(), creationCheck); } else if (Objects.isNull(currentTemplate.version()) || newTemplate.getValue().version() > currentTemplate.version()) { // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can // safely assume it's an old version of the template. @@ -411,7 +411,7 @@ private void addComposableTemplatesIfMissing(ClusterState state) { currentTemplate.version(), newTemplate.getValue().version() ); - putComposableTemplate(state, templateName, newTemplate.getValue(), creationCheck, true); + putComposableTemplate(state, templateName, newTemplate.getValue(), creationCheck); } else { creationCheck.set(false); logger.trace( @@ -433,11 +433,11 @@ private void addComposableTemplatesIfMissing(ClusterState state) { /** * Returns true if the cluster state contains all of the component templates needed by the composable template. If this registry - * requires automatic rollover after index template upgrades (see {@link #applyRolloverAfterTemplateV2Upgrade()}), this method also + * requires automatic rollover after index template upgrades (see {@link #applyRolloverAfterTemplateV2Update()}), this method also * verifies that the installed components templates are of the right version. */ private boolean componentTemplatesInstalled(ClusterState state, ComposableIndexTemplate indexTemplate) { - if (applyRolloverAfterTemplateV2Upgrade() == false) { + if (applyRolloverAfterTemplateV2Update() == false) { // component templates and index templates can be updated independently, we only need to know that the required component // templates are available return state.metadata().componentTemplates().keySet().containsAll(indexTemplate.getRequiredComponentTemplates()); @@ -533,8 +533,7 @@ private void putComposableTemplate( ClusterState state, final String templateName, final ComposableIndexTemplate indexTemplate, - final AtomicBoolean creationCheck, - final boolean isUpgrade + final AtomicBoolean creationCheck ) { final Executor executor = threadPool.generic(); executor.execute(() -> { @@ -549,8 +548,8 @@ private void putComposableTemplate( @Override public void onResponse(AcknowledgedResponse response) { if (response.isAcknowledged()) { - if (isUpgrade && applyRolloverAfterTemplateV2Upgrade()) { - invokeRollover(state, templateName, indexTemplate, creationCheck); + if (applyRolloverAfterTemplateV2Update()) { + invokeRollover(state, templateName, indexTemplate, () -> creationCheck.set((false))); } else { creationCheck.set(false); } @@ -763,12 +762,13 @@ public void onFailure(Exception e) { /** * Allows registries to opt-in for automatic rollover of "relevant" data streams immediately after a composable index template gets - * upgraded. If set to {@code true}, then every time a composable index template is being upgraded, all data streams of which name - * matches this template's index patterns AND of all matching templates the upgraded one has the highest priority, will be rolled over. + * updated, including its initial installation. If set to {@code true}, then every time a composable index template is being updated, + * all data streams of which name matches this template's index patterns AND of all matching templates the upgraded one has the highest + * priority, will be rolled over. * * @return {@code true} if this registry wants to apply automatic rollovers after template V2 upgrades */ - protected boolean applyRolloverAfterTemplateV2Upgrade() { + protected boolean applyRolloverAfterTemplateV2Update() { return false; } @@ -782,50 +782,56 @@ protected void onPutPipelineFailure(String pipelineId, Exception e) { logger.error(() -> format("error adding ingest pipeline template [%s] for [%s]", pipelineId, getOrigin()), e); } + /** + * invokeRollover rolls over any data streams matching the index template, + * and then invokes runAfter. + */ private void invokeRollover( final ClusterState state, final String templateName, final ComposableIndexTemplate indexTemplate, - final AtomicBoolean creationCheck + final Runnable runAfter ) { final Executor executor = threadPool.generic(); executor.execute(() -> { List rolloverTargets = findRolloverTargetDataStreams(state, templateName, indexTemplate); - if (rolloverTargets.isEmpty() == false) { - GroupedActionListener groupedActionListener = new GroupedActionListener<>( - rolloverTargets.size(), - new ActionListener<>() { - @Override - public void onResponse(Collection rolloverResponses) { - creationCheck.set(false); - onRolloversBulkResponse(rolloverResponses); - } + if (rolloverTargets.isEmpty()) { + runAfter.run(); + return; + } + GroupedActionListener groupedActionListener = new GroupedActionListener<>( + rolloverTargets.size(), + new ActionListener<>() { + @Override + public void onResponse(Collection rolloverResponses) { + runAfter.run(); + onRolloversBulkResponse(rolloverResponses); + } - @Override - public void onFailure(Exception e) { - creationCheck.set(false); - onRolloverFailure(e); - } + @Override + public void onFailure(Exception e) { + runAfter.run(); + onRolloverFailure(e); } - ); - for (String rolloverTarget : rolloverTargets) { - logger.info( - "rolling over data stream [{}] lazily as a followup to the upgrade of the [{}] index template [{}]", - rolloverTarget, - getOrigin(), - templateName - ); - RolloverRequest request = new RolloverRequest(rolloverTarget, null); - request.lazy(true); - request.masterNodeTimeout(TimeValue.MAX_VALUE); - executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - getOrigin(), - request, - groupedActionListener, - (req, listener) -> client.execute(RolloverAction.INSTANCE, req, listener) - ); } + ); + for (String rolloverTarget : rolloverTargets) { + logger.info( + "rolling over data stream [{}] lazily as a followup to the upgrade of the [{}] index template [{}]", + rolloverTarget, + getOrigin(), + templateName + ); + RolloverRequest request = new RolloverRequest(rolloverTarget, null); + request.lazy(true); + request.masterNodeTimeout(TimeValue.MAX_VALUE); + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + getOrigin(), + request, + groupedActionListener, + (req, listener) -> client.execute(RolloverAction.INSTANCE, req, listener) + ); } }); } @@ -865,7 +871,21 @@ static List findRolloverTargetDataStreams(ClusterState state, String tem .stream() // Limit to checking data streams that match any of the index template's index patterns .filter(ds -> indexTemplate.indexPatterns().stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName()))) - .filter(ds -> templateName.equals(MetadataIndexTemplateService.findV2Template(metadata, ds.getName(), ds.isHidden()))) + .filter(ds -> { + final String dsTemplateName = MetadataIndexTemplateService.findV2Template(metadata, ds.getName(), ds.isHidden()); + if (templateName.equals(dsTemplateName)) { + return true; + } + // findV2Template did not match templateName, which implies one of two things: + // - indexTemplate has a lower priority than the index template matching for ds, OR + // - indexTemplate does not yet exist in cluster state (i.e. because it's in the process of being + // installed or updated) + // + // Because of the second case, we must check if indexTemplate's priority is greater than the matching + // index template, in case it would take precedence after installation/update. + final ComposableIndexTemplate dsTemplate = metadata.templatesV2().get(dsTemplateName); + return dsTemplate == null || indexTemplate.priorityOrZero() > dsTemplate.priorityOrZero(); + }) .map(DataStream::getName) .collect(Collectors.toList()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java new file mode 100644 index 0000000000000..c8ddd46c5912f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -0,0 +1,233 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.template; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.yaml.YamlXContent; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.core.template.ResourceUtils.loadResource; +import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8; + +/** + * Creates index templates and ingest pipelines based on YAML files defined in resources.yaml. + */ +public abstract class YamlTemplateRegistry extends IndexTemplateRegistry { + private static final Logger logger = LogManager.getLogger(YamlTemplateRegistry.class); + // this node feature is a redefinition of {@link DataStreamFeatures#DATA_STREAM_LIFECYCLE} and it's meant to avoid adding a + // dependency to the data-streams module just for this + public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle"); + private final int version; + + private final Map componentTemplates; + private final Map composableIndexTemplates; + private final List ingestPipelines; + private final FeatureService featureService; + private volatile boolean enabled; + + public YamlTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry, + FeatureService featureService + ) { + this(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService, ignored -> true); + } + + @SuppressWarnings({ "unchecked", "this-escape" }) + public YamlTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry, + FeatureService featureService, + Predicate templateFilter + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + try { + final Map resources = XContentHelper.convertToMap( + YamlXContent.yamlXContent, + loadResource(this.getClass(), "/resources.yaml"), + false + ); + version = (((Number) resources.get("version")).intValue()); + + final List componentTemplateNames = (List) resources.get("component-templates"); + final List indexTemplateNames = (List) resources.get("index-templates"); + final List ingestPipelineConfigs = (List) resources.get("ingest-pipelines"); + + componentTemplates = Optional.ofNullable(componentTemplateNames) + .orElse(Collections.emptyList()) + .stream() + .map(o -> (String) o) + .filter(templateFilter) + .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version))); + composableIndexTemplates = Optional.ofNullable(indexTemplateNames) + .orElse(Collections.emptyList()) + .stream() + .map(o -> (String) o) + .filter(templateFilter) + .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version))); + ingestPipelines = Optional.ofNullable(ingestPipelineConfigs) + .orElse(Collections.emptyList()) + .stream() + .map(o -> (Map>) o) + .map(map -> { + Map.Entry> pipelineConfig = map.entrySet().iterator().next(); + return loadIngestPipeline( + pipelineConfig.getKey(), + version, + (List) pipelineConfig.getValue().get("dependencies") + ); + }) + .collect(Collectors.toList()); + this.featureService = featureService; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public int getVersion() { + return version; + } + + /*** + * + * @return A friendly, human-readable name of the index template registry + */ + public abstract String getName(); + + public void setEnabled(boolean enabled) { + logger.info("{} index template registry is {}", getName(), enabled ? "enabled" : "disabled"); + this.enabled = enabled; + } + + public boolean isEnabled() { + return enabled; + } + + public void close() { + clusterService.removeListener(this); + } + + @Override + protected boolean isClusterReady(ClusterChangedEvent event) { + // Ensure current version of the components are installed only after versions that support data stream lifecycle + // due to the use of the feature in all the `@lifecycle` component templates + return featureService.clusterHasFeature(event.state(), DATA_STREAM_LIFECYCLE); + } + + @Override + protected boolean requiresMasterNode() { + return true; + } + + @Override + public Map getComponentTemplateConfigs() { + if (enabled) { + return componentTemplates; + } else { + return Map.of(); + } + } + + @Override + public Map getComposableTemplateConfigs() { + if (enabled) { + return composableIndexTemplates; + } else { + return Map.of(); + } + } + + @Override + public List getIngestPipelines() { + if (enabled) { + return ingestPipelines; + } else { + return Collections.emptyList(); + } + } + + protected abstract String getVersionProperty(); + + private ComponentTemplate loadComponentTemplate(String name, int version) { + try { + final byte[] content = loadVersionedResourceUTF8( + this.getClass(), + "/component-templates/" + name + ".yaml", + version, + getVersionProperty() + ); + try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { + return ComponentTemplate.parse(parser); + } + } catch (Exception e) { + throw new RuntimeException("failed to load " + getName() + " Ingest plugin's component template: " + name, e); + } + } + + private ComposableIndexTemplate loadIndexTemplate(String name, int version) { + try { + final byte[] content = loadVersionedResourceUTF8( + this.getClass(), + "/index-templates/" + name + ".yaml", + version, + getVersionProperty() + ); + try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { + return ComposableIndexTemplate.parse(parser); + } + } catch (Exception e) { + throw new RuntimeException("failed to load " + getName() + " Ingest plugin's index template: " + name, e); + } + } + + private IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List dependencies) { + if (dependencies == null) { + dependencies = Collections.emptyList(); + } + return new YamlIngestPipelineConfig( + name, + "/ingest-pipelines/" + name + ".yaml", + version, + getVersionProperty(), + dependencies, + this.getClass() + ); + } + + @Override + protected boolean applyRolloverAfterTemplateV2Update() { + return true; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java index b0127c0005323..8d8aa9fd3c634 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java @@ -435,7 +435,7 @@ public void testAutomaticRollover() throws Exception { assertThat(suppressed[0].getMessage(), startsWith("Failed to rollover logs-my_app-")); } - public void testNoRolloverForFreshInstalledIndexTemplate() throws Exception { + public void testRolloverForFreshInstalledIndexTemplate() throws Exception { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); @@ -473,9 +473,10 @@ public void testNoRolloverForFreshInstalledIndexTemplate() throws Exception { registry.setApplyRollover(true); registry.clusterChanged(event); assertBusy(() -> assertThat(putIndexTemplateCounter.get(), equalTo(1))); - // the index component is first installed, not upgraded, therefore rollover should not be triggered + // rollover should be triggered even for the first installation, since the template + // may now take precedence over a data stream's existing index template Thread.sleep(100L); - assertThat(rolloverCounter.get(), equalTo(0)); + assertThat(rolloverCounter.get(), equalTo(2)); } public void testThatTemplatesAreNotUpgradedWhenNotNeeded() throws Exception { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TestRegistryWithCustomPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TestRegistryWithCustomPlugin.java index 349fdfe1259c9..2ef0c7f5301ec 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TestRegistryWithCustomPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TestRegistryWithCustomPlugin.java @@ -118,7 +118,7 @@ public void setPolicyUpgradeRequired(boolean policyUpgradeRequired) { } @Override - protected boolean applyRolloverAfterTemplateV2Upgrade() { + protected boolean applyRolloverAfterTemplateV2Update() { return applyRollover.get(); } From 7e4757d16d768f4fb12d7bf0c5125151639d8ba5 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Tue, 5 Nov 2024 11:40:41 +0000 Subject: [PATCH 2/3] Fix function name based on cherry-picked commit --- .../elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 04b0257f4180a..578241f07c9a3 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -188,7 +188,7 @@ private static IngestPipelineConfig loadIngestPipeline(String name, int version, } @Override - protected boolean applyRolloverAfterTemplateV2Upgrade() { + protected boolean applyRolloverAfterTemplateV2Update() { return true; } } From c13ab08d67fa5775ed073786f0fc6a2e46dfe6d8 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Tue, 5 Nov 2024 11:56:41 +0000 Subject: [PATCH 3/3] Remove unnecessary cherrypick file from 8.15 --- .../core/template/YamlTemplateRegistry.java | 233 ------------------ 1 file changed, 233 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java deleted file mode 100644 index c8ddd46c5912f..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.template; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.FeatureService; -import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.yaml.YamlXContent; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static org.elasticsearch.xpack.core.template.ResourceUtils.loadResource; -import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8; - -/** - * Creates index templates and ingest pipelines based on YAML files defined in resources.yaml. - */ -public abstract class YamlTemplateRegistry extends IndexTemplateRegistry { - private static final Logger logger = LogManager.getLogger(YamlTemplateRegistry.class); - // this node feature is a redefinition of {@link DataStreamFeatures#DATA_STREAM_LIFECYCLE} and it's meant to avoid adding a - // dependency to the data-streams module just for this - public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle"); - private final int version; - - private final Map componentTemplates; - private final Map composableIndexTemplates; - private final List ingestPipelines; - private final FeatureService featureService; - private volatile boolean enabled; - - public YamlTemplateRegistry( - Settings nodeSettings, - ClusterService clusterService, - ThreadPool threadPool, - Client client, - NamedXContentRegistry xContentRegistry, - FeatureService featureService - ) { - this(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService, ignored -> true); - } - - @SuppressWarnings({ "unchecked", "this-escape" }) - public YamlTemplateRegistry( - Settings nodeSettings, - ClusterService clusterService, - ThreadPool threadPool, - Client client, - NamedXContentRegistry xContentRegistry, - FeatureService featureService, - Predicate templateFilter - ) { - super(nodeSettings, clusterService, threadPool, client, xContentRegistry); - try { - final Map resources = XContentHelper.convertToMap( - YamlXContent.yamlXContent, - loadResource(this.getClass(), "/resources.yaml"), - false - ); - version = (((Number) resources.get("version")).intValue()); - - final List componentTemplateNames = (List) resources.get("component-templates"); - final List indexTemplateNames = (List) resources.get("index-templates"); - final List ingestPipelineConfigs = (List) resources.get("ingest-pipelines"); - - componentTemplates = Optional.ofNullable(componentTemplateNames) - .orElse(Collections.emptyList()) - .stream() - .map(o -> (String) o) - .filter(templateFilter) - .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version))); - composableIndexTemplates = Optional.ofNullable(indexTemplateNames) - .orElse(Collections.emptyList()) - .stream() - .map(o -> (String) o) - .filter(templateFilter) - .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version))); - ingestPipelines = Optional.ofNullable(ingestPipelineConfigs) - .orElse(Collections.emptyList()) - .stream() - .map(o -> (Map>) o) - .map(map -> { - Map.Entry> pipelineConfig = map.entrySet().iterator().next(); - return loadIngestPipeline( - pipelineConfig.getKey(), - version, - (List) pipelineConfig.getValue().get("dependencies") - ); - }) - .collect(Collectors.toList()); - this.featureService = featureService; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public int getVersion() { - return version; - } - - /*** - * - * @return A friendly, human-readable name of the index template registry - */ - public abstract String getName(); - - public void setEnabled(boolean enabled) { - logger.info("{} index template registry is {}", getName(), enabled ? "enabled" : "disabled"); - this.enabled = enabled; - } - - public boolean isEnabled() { - return enabled; - } - - public void close() { - clusterService.removeListener(this); - } - - @Override - protected boolean isClusterReady(ClusterChangedEvent event) { - // Ensure current version of the components are installed only after versions that support data stream lifecycle - // due to the use of the feature in all the `@lifecycle` component templates - return featureService.clusterHasFeature(event.state(), DATA_STREAM_LIFECYCLE); - } - - @Override - protected boolean requiresMasterNode() { - return true; - } - - @Override - public Map getComponentTemplateConfigs() { - if (enabled) { - return componentTemplates; - } else { - return Map.of(); - } - } - - @Override - public Map getComposableTemplateConfigs() { - if (enabled) { - return composableIndexTemplates; - } else { - return Map.of(); - } - } - - @Override - public List getIngestPipelines() { - if (enabled) { - return ingestPipelines; - } else { - return Collections.emptyList(); - } - } - - protected abstract String getVersionProperty(); - - private ComponentTemplate loadComponentTemplate(String name, int version) { - try { - final byte[] content = loadVersionedResourceUTF8( - this.getClass(), - "/component-templates/" + name + ".yaml", - version, - getVersionProperty() - ); - try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { - return ComponentTemplate.parse(parser); - } - } catch (Exception e) { - throw new RuntimeException("failed to load " + getName() + " Ingest plugin's component template: " + name, e); - } - } - - private ComposableIndexTemplate loadIndexTemplate(String name, int version) { - try { - final byte[] content = loadVersionedResourceUTF8( - this.getClass(), - "/index-templates/" + name + ".yaml", - version, - getVersionProperty() - ); - try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { - return ComposableIndexTemplate.parse(parser); - } - } catch (Exception e) { - throw new RuntimeException("failed to load " + getName() + " Ingest plugin's index template: " + name, e); - } - } - - private IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List dependencies) { - if (dependencies == null) { - dependencies = Collections.emptyList(); - } - return new YamlIngestPipelineConfig( - name, - "/ingest-pipelines/" + name + ".yaml", - version, - getVersionProperty(), - dependencies, - this.getClass() - ); - } - - @Override - protected boolean applyRolloverAfterTemplateV2Update() { - return true; - } -}