Skip to content

Commit

Permalink
[7.11] require master node for watcher template (#69859)
Browse files Browse the repository at this point in the history 
Backporting #69147 to 7.11 branch.

This commit requires Watcher to install composable index templates
only from the master node since during a rolling upgrade it could
try to talk to a master node that predates composable index templates.

Some tests have also been adjusted to that allows for this change.
Those test changes have been discussed offline and can generally be
summarized as the things they are testing is an edge case where if that
edge case (breaking changes to the mappings of watcher history index) between
versions then there would be other concerns beyond the need for which
node installs the watcher templates.

closes #66837

Co-authored-by: Martijn van Groningen <martijn.v.groningen@gmail.com>
Co-authored-by: Jake Landis <jake.landis@elastic.co>
  • Loading branch information
@martijnvg @jakelandis
martijnvg and jakelandis committed Mar 3, 2021
1 parent ce1c84f commit d7ba942
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 75 deletions.
30 changes: 16 additions & 14 deletions ...r/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;

import static org.elasticsearch.xpack.core.ClientHelper.WATCHER_ORIGIN;

Expand Down Expand Up @@ -135,22 +136,23 @@ protected String getOrigin() {
}

public static boolean validate(ClusterState state) {
if(state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_9_0)){
return (state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) ||
state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) &&
state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) &&
state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME);
} else if (state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) {
return (state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_11) ||
state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_11)) &&
state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME_11) &&
state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME_11);
final Stream<String> watcherHistoryTemplateIds;
if (state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_9_0)){
watcherHistoryTemplateIds = state.getMetadata().templatesV2().keySet().stream();
} else {
return (state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10) ||
state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_10)) &&
state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME_11) &&
state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME_11);
watcherHistoryTemplateIds = Arrays.stream(state.getMetadata().getTemplates().keys().toArray(String.class));
}
return watcherHistoryTemplateIds.filter(s -> s.startsWith(".watch-history-"))
.map(s -> Integer.valueOf(s.substring(s.lastIndexOf('-') + 1)))
.anyMatch(version -> version >= 9);
}


@Override
protected boolean requiresMasterNode() {
// These installs a composable index template which is only supported in early versions of 7.x
// In mixed cluster without this set to true can result in errors in the logs during rolling upgrades.
// If these template(s) are only installed via elected master node then composable templates are available.
return true;
}
}
60 changes: 0 additions & 60 deletions .../test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import static org.elasticsearch.mock.orig.Mockito.verify;
Expand All @@ -70,7 +69,6 @@
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.same;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
Expand Down Expand Up @@ -256,64 +254,6 @@ public void testThatTemplatesExist() {
}
}

// if a node is newer than the master node, the template needs to be applied as well
// otherwise a rolling upgrade would not work as expected, when the node has a .watches shard on it
public void testThatTemplatesAreAppliedOnNewerNodes() {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build();

Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(".watch-history-6", 6);
ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes);
registry.clusterChanged(event);

ArgumentCaptor<PutComposableIndexTemplateAction.Request> argumentCaptor =
ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class);
verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject());
assertTrue(argumentCaptor.getAllValues().stream()
.anyMatch(r -> r.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME)));
}

public void testThatTemplatesWithHiddenAreAppliedOnNewerNodes() {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("master").masterNodeId("master").add(node).add(masterNode).build();

Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(".watch-history-6", 6);
ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes);
registry.clusterChanged(event);

ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client.admin().indices(), atLeastOnce()).putTemplate(argumentCaptor.capture(), anyObject());
assertTrue(argumentCaptor.getAllValues().stream()
.anyMatch(i -> i.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10)));

existingTemplates.remove(".watch-history-6");
existingTemplates.put(".watch-history-10", 10);
masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
nodes = DiscoveryNodes.builder().localNodeId("master").masterNodeId("master").add(masterNode).add(node).build();
event = createClusterChangedEvent(existingTemplates, nodes);
registry.clusterChanged(event);

argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client.admin().indices(), atLeastOnce()).putTemplate(argumentCaptor.capture(), anyObject());
assertTrue(argumentCaptor.getAllValues().stream()
.anyMatch(i -> i.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10)));
ArgumentCaptor<PutComposableIndexTemplateAction.Request> captor =
ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class);
verify(client, atLeastOnce()).execute(same(PutComposableIndexTemplateAction.INSTANCE), captor.capture(), anyObject());
Set<String> templateNames =
captor.getAllValues().stream().map(PutComposableIndexTemplateAction.Request::name).collect(Collectors.toSet());
assertTrue(templateNames.contains(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME));
assertFalse(templateNames.contains(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10));
}

public void testThatTemplatesAreNotAppliedOnSameVersionNodes() {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
Expand Down
4 changes: 3 additions & 1 deletion x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ public void testWatcherRestart() throws Exception {
client().performRequest(new Request("POST", "/_watcher/_start"));
ensureWatcherStarted();

validateHistoryTemplate();
if (CLUSTER_TYPE.equals(ClusterType.UPGRADED)) {
validateHistoryTemplate();
}
}

private void validateHistoryTemplate() throws Exception {
Expand Down

0 comments on commit d7ba942

Please sign in to comment.