Add support for index pattern exclusion in CCR AutoFollow (#73765)

This commit adds the ability to specify exclusion patterns in Auto-Follow patterns.
This allows excluding indices that match any of the inclusion patterns and also match
some of the exclusion patterns giving more fine grained control in scenarios where this is important.

Related #67686
Backport of #72935

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java

@@ -82,11 +82,19 @@ public static class Pattern extends FollowConfig {
 
         @SuppressWarnings("unchecked")
         private static final ConstructingObjectParser<Pattern, Void> PARSER = new ConstructingObjectParser<>(
-            "pattern", true, args -> new Pattern((String) args[0], (List<String>) args[1], (String) args[2]));
+            "pattern",
+            true,
+            args -> new Pattern((String) args[0],
+                                (List<String>) args[1],
+                                args[2] == null ? Collections.emptyList() : (List<String>) args[2],
+                                (String) args[3])
+        );
 
         static {
             PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);
             PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD);
+            PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(),
+                                      PutAutoFollowPatternRequest.LEADER_EXCLUSION_PATTERNS_FIELD);
             PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD);
             PARSER.declareObject(Pattern::setSettings, (p, c) -> Settings.fromXContent(p), PutAutoFollowPatternRequest.SETTINGS);
             PARSER.declareInt(Pattern::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT);
@@ -123,11 +131,16 @@ public static class Pattern extends FollowConfig {
 
         private final String remoteCluster;
         private final List<String> leaderIndexPatterns;
+        private final List<String> leaderIndexExclusionPatterns;
         private final String followIndexNamePattern;
 
-        Pattern(String remoteCluster, List<String> leaderIndexPatterns, String followIndexNamePattern) {
+        Pattern(String remoteCluster,
+                List<String> leaderIndexPatterns,
+                List<String> leaderIndexExclusionPatterns,
+                String followIndexNamePattern) {
             this.remoteCluster = remoteCluster;
             this.leaderIndexPatterns = leaderIndexPatterns;
+            this.leaderIndexExclusionPatterns = leaderIndexExclusionPatterns;
             this.followIndexNamePattern = followIndexNamePattern;
         }
 
@@ -139,6 +152,10 @@ public List<String> getLeaderIndexPatterns() {
             return leaderIndexPatterns;
         }
 
+        public List<String> getLeaderIndexExclusionPatterns() {
+            return leaderIndexExclusionPatterns;
+        }
+
         public String getFollowIndexNamePattern() {
             return followIndexNamePattern;
         }
@@ -151,6 +168,7 @@ public boolean equals(Object o) {
             Pattern pattern = (Pattern) o;
             return Objects.equals(remoteCluster, pattern.remoteCluster) &&
                 Objects.equals(leaderIndexPatterns, pattern.leaderIndexPatterns) &&
+                Objects.equals(leaderIndexExclusionPatterns, pattern.leaderIndexExclusionPatterns) &&
                 Objects.equals(followIndexNamePattern, pattern.followIndexNamePattern);
         }
 
@@ -160,6 +178,7 @@ public int hashCode() {
                 super.hashCode(),
                 remoteCluster,
                 leaderIndexPatterns,
+                leaderIndexExclusionPatterns,
                 followIndexNamePattern
             );
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java

@@ -14,23 +14,36 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 public final class PutAutoFollowPatternRequest extends FollowConfig implements Validatable, ToXContentObject {
 
     static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns");
+    static final ParseField LEADER_EXCLUSION_PATTERNS_FIELD = new ParseField("leader_index_exclusion_patterns");
     static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern");
 
     private final String name;
     private final String remoteCluster;
     private final List<String> leaderIndexPatterns;
+    private final List<String> leaderIndexExclusionPatterns;
     private String followIndexNamePattern;
 
-    public PutAutoFollowPatternRequest(String name, String remoteCluster, List<String> leaderIndexPatterns) {
+    public PutAutoFollowPatternRequest(String name,
+                                       String remoteCluster,
+                                       List<String> leaderIndexPatterns) {
+        this(name, remoteCluster, leaderIndexPatterns, Collections.emptyList());
+    }
+
+    public PutAutoFollowPatternRequest(String name,
+                                       String remoteCluster,
+                                       List<String> leaderIndexPatterns,
+                                       List<String> leaderIndexExclusionPatterns) {
         this.name = Objects.requireNonNull(name);
         this.remoteCluster = Objects.requireNonNull(remoteCluster);
         this.leaderIndexPatterns = Objects.requireNonNull(leaderIndexPatterns);
+        this.leaderIndexExclusionPatterns = Objects.requireNonNull(leaderIndexExclusionPatterns);
     }
 
     public String getName() {
@@ -45,6 +58,10 @@ public List<String> getLeaderIndexPatterns() {
         return leaderIndexPatterns;
     }
 
+    public List<String> getLeaderIndexExclusionPatterns() {
+        return leaderIndexExclusionPatterns;
+    }
+
     public String getFollowIndexNamePattern() {
         return followIndexNamePattern;
     }
@@ -58,6 +75,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(PutFollowRequest.REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster);
         builder.field(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns);
+        if (leaderIndexExclusionPatterns.isEmpty() == false) {
+            builder.field(LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns);
+        }
         if (followIndexNamePattern != null) {
             builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern);
         }
@@ -75,6 +95,7 @@ public boolean equals(Object o) {
         return Objects.equals(name, that.name) &&
             Objects.equals(remoteCluster, that.remoteCluster) &&
             Objects.equals(leaderIndexPatterns, that.leaderIndexPatterns) &&
+            Objects.equals(leaderIndexExclusionPatterns, that.leaderIndexExclusionPatterns) &&
             Objects.equals(followIndexNamePattern, that.followIndexNamePattern);
     }
 
@@ -85,6 +106,7 @@ public int hashCode() {
             name,
             remoteCluster,
             leaderIndexPatterns,
+            leaderIndexExclusionPatterns,
             followIndexNamePattern
         );
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java

@@ -58,6 +58,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class CCRIT extends ESRestHighLevelClientTestCase {
 
@@ -245,8 +246,10 @@ public void testForgetFollower() throws IOException {
 
     public void testAutoFollowing() throws Exception {
         CcrClient ccrClient = highLevelClient().ccr();
-        PutAutoFollowPatternRequest putAutoFollowPatternRequest =
-            new PutAutoFollowPatternRequest("pattern1", "local_cluster", Collections.singletonList("logs-*"));
+        PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("pattern1",
+                "local_cluster",
+                Collections.singletonList("logs-*"),
+                Collections.singletonList("logs-excluded"));
         putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}");
         final int followerNumberOfReplicas = randomIntBetween(0, 4);
         final Settings autoFollowerPatternSettings =
@@ -256,6 +259,11 @@ public void testAutoFollowing() throws Exception {
             execute(putAutoFollowPatternRequest, ccrClient::putAutoFollowPattern, ccrClient::putAutoFollowPatternAsync);
         assertThat(putAutoFollowPatternResponse.isAcknowledged(), is(true));
 
+        CreateIndexRequest createExcludedIndexRequest = new CreateIndexRequest("logs-excluded");
+        CreateIndexResponse createExcludedIndexResponse =
+            highLevelClient().indices().create(createExcludedIndexRequest, RequestOptions.DEFAULT);
+        assertThat(createExcludedIndexResponse.isAcknowledged(), is(true));
+
         CreateIndexRequest createIndexRequest = new CreateIndexRequest("logs-20200101");
         CreateIndexResponse response = highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
         assertThat(response.isAcknowledged(), is(true));
@@ -265,11 +273,13 @@ public void testAutoFollowing() throws Exception {
             CcrStatsResponse ccrStatsResponse = execute(ccrStatsRequest, ccrClient::getCcrStats, ccrClient::getCcrStatsAsync);
             assertThat(ccrStatsResponse.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(1L));
             assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-20200101"), notNullValue());
+            assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-excluded"), nullValue());
         });
         assertThat(indexExists("copy-logs-20200101"), is(true));
         assertThat(
             getIndexSettingsAsMap("copy-logs-20200101"),
             hasEntry("index.number_of_replicas", Integer.toString(followerNumberOfReplicas)));
+        assertThat(indexExists("copy-logs-excluded"), is(false));
 
         GetAutoFollowPatternRequest getAutoFollowPatternRequest =
             randomBoolean() ? new GetAutoFollowPatternRequest("pattern1") : new GetAutoFollowPatternRequest();
@@ -280,6 +290,7 @@ public void testAutoFollowing() throws Exception {
         assertThat(pattern, notNullValue());
         assertThat(pattern.getRemoteCluster(), equalTo(putAutoFollowPatternRequest.getRemoteCluster()));
         assertThat(pattern.getLeaderIndexPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexPatterns()));
+        assertThat(pattern.getLeaderIndexExclusionPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexExclusionPatterns()));
         assertThat(pattern.getFollowIndexNamePattern(), equalTo(putAutoFollowPatternRequest.getFollowIndexNamePattern()));
         assertThat(pattern.getSettings(), equalTo(autoFollowerPatternSettings));
 

client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java

@@ -101,7 +101,10 @@ public void testForgetFollower() throws IOException {
 
     public void testPutAutofollowPattern() throws Exception {
         PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest(randomAlphaOfLength(4),
-            randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
+            randomAlphaOfLength(4),
+            Arrays.asList(generateRandomStringArray(4, 4, false)),
+            Arrays.asList(generateRandomStringArray(4, 4, false))
+        );
         if (randomBoolean()) {
             putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
         }

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java

@@ -39,6 +39,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
         for (int i = 0; i < numPatterns; i++) {
             String remoteCluster = randomAlphaOfLength(4);
             List<String> leaderIndexPatterns = Collections.singletonList(randomAlphaOfLength(4));
+            List<String> leaderIndexExclusionsPatterns = randomList(0, randomIntBetween(1, 10), () -> randomAlphaOfLength(4));
             String followIndexNamePattern = randomAlphaOfLength(4);
             final Settings settings =
                 Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build();
@@ -89,6 +90,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
                 new AutoFollowMetadata.AutoFollowPattern(
                     remoteCluster,
                     leaderIndexPatterns,
+                    leaderIndexExclusionsPatterns,
                     followIndexNamePattern,
                     settings,
                     active,
@@ -124,6 +126,7 @@ protected void assertInstances(GetAutoFollowPatternAction.Response serverTestIns
             assertThat(serverPattern.getRemoteCluster(), equalTo(clientPattern.getRemoteCluster()));
             assertThat(serverPattern.getLeaderIndexPatterns(), equalTo(clientPattern.getLeaderIndexPatterns()));
             assertThat(serverPattern.getFollowIndexPattern(), equalTo(clientPattern.getFollowIndexNamePattern()));
+            assertThat(serverPattern.getLeaderIndexExclusionPatterns(), equalTo(clientPattern.getLeaderIndexExclusionPatterns()));
             assertThat(serverPattern.getSettings(), equalTo(clientPattern.getSettings()));
             assertThat(serverPattern.getMaxOutstandingReadRequests(), equalTo(clientPattern.getMaxOutstandingReadRequests()));
             assertThat(serverPattern.getMaxOutstandingWriteRequests(), equalTo(clientPattern.getMaxOutstandingWriteRequests()));

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java

@@ -28,7 +28,10 @@ public class PutAutoFollowPatternRequestTests extends AbstractRequestTestCase<
     protected PutAutoFollowPatternRequest createClientTestInstance() {
         // Name isn't serialized, because it specified in url path, so no need to randomly generate it here.
         PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("name",
-            randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
+            randomAlphaOfLength(4),
+            Arrays.asList(generateRandomStringArray(4, 4, false)),
+            Arrays.asList(generateRandomStringArray(4, 4, false))
+        );
         if (randomBoolean()) {
             putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
         }
@@ -75,6 +78,7 @@ protected void assertInstances(PutAutoFollowPatternAction.Request serverInstance
         assertThat(serverInstance.getName(), equalTo(clientTestInstance.getName()));
         assertThat(serverInstance.getRemoteCluster(), equalTo(clientTestInstance.getRemoteCluster()));
         assertThat(serverInstance.getLeaderIndexPatterns(), equalTo(clientTestInstance.getLeaderIndexPatterns()));
+        assertThat(serverInstance.getLeaderIndexExclusionPatterns(), equalTo(clientTestInstance.getLeaderIndexExclusionPatterns()));
         assertThat(serverInstance.getFollowIndexNamePattern(), equalTo(clientTestInstance.getFollowIndexNamePattern()));
         assertFollowConfig(serverInstance.getParameters(), clientTestInstance);
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java

@@ -476,12 +476,13 @@ public void testPutAutoFollowPattern() throws Exception {
             new PutAutoFollowPatternRequest(
                 "my_pattern", // <1>
                 "local", // <2>
-                Arrays.asList("logs-*", "metrics-*") // <3>
+                Arrays.asList("logs-*", "metrics-*"), // <3>
+                Arrays.asList("logs-excluded", "metrics-excluded") // <4>
         );
-        request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4>
+        request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <5>
         Settings settings =
             Settings.builder().put("index.number_of_replicas", 0L).build();
-        request.setSettings(settings); // <5>
+        request.setSettings(settings); // <6>
         // end::ccr-put-auto-follow-pattern-request
 
         // tag::ccr-put-auto-follow-pattern-execute

docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc

@@ -21,8 +21,9 @@ include-tagged::{doc-tests-file}[{api}-request]
 <1> The name of the auto follow pattern.
 <2> The name of the remote cluster.
 <3> The leader index patterns.
-<4> The pattern used to create the follower index
-<5> The settings overrides for the follower index
+<4> The leader index exclusion patterns.
+<5> The pattern used to create the follower index.
+<6> The settings overrides for the follower index.
 
 [id="{upid}-{api}-response"]
 ==== Response

docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc

@@ -22,6 +22,10 @@ PUT /_ccr/auto_follow/my_auto_follow_pattern
   [
     "leader_index*"
   ],
+  "leader_index_exclusion_patterns":
+  [
+    "leader_index_001"
+  ],
   "follow_index_pattern" : "{{leader_index}}-follower"
 }
 --------------------------------------------------
@@ -96,6 +100,10 @@ The API returns the following result:
         [
           "leader_index*"
         ],
+        "leader_index_exclusion_patterns":
+        [
+          "leader_index_001"
+        ],
         "follow_index_pattern" : "{{leader_index}}-follower"
       }
     }

docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc

@@ -20,13 +20,18 @@ PUT /_ccr/auto_follow/<auto_follow_pattern_name>
   [
     "<leader_index_pattern>"
   ],
+  "leader_index_exclusion_patterns":
+  [
+    "<leader_index_exclusion_pattern>"
+  ],
   "follow_index_pattern" : "<follow_index_pattern>"
 }
 --------------------------------------------------
 // TEST[setup:remote_cluster]
 // TEST[s/<auto_follow_pattern_name>/auto_follow_pattern_name/]
 // TEST[s/<remote_cluster>/remote_cluster/]
 // TEST[s/<leader_index_patterns>/leader_index*/]
+// TEST[s/<leader_index_exclusion_pattern>//]
 // TEST[s/<follow_index_pattern>/{{leader_index}}-follower/]
 
 //////////////////////////
@@ -54,7 +59,10 @@ This API creates a new named collection of
 <<ccr-auto-follow,auto-follow patterns>> against the remote cluster
 specified in the request body. Newly created indices on the remote cluster
 matching any of the specified patterns will be automatically configured as follower
-indices.
+indices. Additionally, this API can be used to update existing
+<<ccr-auto-follow,auto-follow patterns>>. Note that follower indices that were configured automatically
+before updating an auto-follow pattern will remain unchanged even if they don't match against
+the new patterns.
 
 [[ccr-put-auto-follow-pattern-path-parms]]
 ==== {api-path-parms-title}
@@ -72,6 +80,11 @@ indices.
   (Optional, array) An array of simple index patterns to match against indices
   in the remote cluster specified by the `remote_cluster` field.
 
+`leader_index_exclusion_patterns`::
+  (Optional, array) An array of simple index patterns that can be used to exclude indices
+  from being auto-followed. Indices in the remote cluster whose names are matching one or
+  more `leader_index_patterns` and one or more `leader_index_exclusion_patterns` won't be followed.
+
 `follow_index_pattern`::
   (Optional, string) The name of follower index. The template `{{leader_index}}`
   can be used to derive the name of the follower index from the name of the