[Connector API] Support detached index for connectors (#106236)

elastic · Mar 14, 2024 · d1cf3b1 · d1cf3b1
1 parent 1b8f2cf
commit d1cf3b1
Show file tree

Hide file tree

Showing 12 changed files with 182 additions and 70 deletions.
diff --git a/...rch/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/300_connector_put.yml b/...rch/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/300_connector_put.yml
@@ -135,6 +135,22 @@ setup:
         body:
           index_name: search-test
 
+---
+'Create Connector - Without index attached':
+  - do:
+      connector.put:
+        connector_id: test-connector-1
+        body:
+          service_type: super-connector
+
+  - match: { result: 'created' }
+
+  - do:
+      connector.get:
+        connector_id: test-connector-1
+
+  - match: { index_name: null }
+  - match: { service_type: super-connector }
 
 ---
 "Put connector fails for unprivileged user":

diff --git a/...ch/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/305_connector_post.yml b/...ch/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/305_connector_post.yml
@@ -105,6 +105,23 @@ setup:
           index_name: search-test
 
 
+---
+'Create Connector - Without index attached':
+  - do:
+      connector.post:
+        body:
+          service_type: super-connector
+
+  - set:  { id: id }
+  - match: { id: $id }
+
+  - do:
+      connector.get:
+        connector_id: $id
+
+  - match: { index_name: null }
+  - match: { service_type: super-connector }
+
 ---
 "Post connector fails for unprivileged user":
   - skip:

diff --git a/...c/yamlRestTest/resources/rest-api-spec/test/entsearch/338_connector_update_index_name.yml b/...c/yamlRestTest/resources/rest-api-spec/test/entsearch/338_connector_update_index_name.yml
@@ -31,6 +31,23 @@ setup:
   - match: { index_name: search-2-test }
   - match: { status: created }
 
+---
+"Update Connector Index Name - detach index":
+  - do:
+      connector.update_index_name:
+        connector_id: test-connector
+        body:
+          index_name: null
+
+
+  - match: { result: updated }
+
+  - do:
+      connector.get:
+        connector_id: test-connector
+
+  - match: { index_name: null }
+
 ---
 "Update Connector Index Name - 404 when connector doesn't exist":
   - do:

diff --git a/...t/src/yamlRestTest/resources/rest-api-spec/test/entsearch/400_connector_sync_job_post.yml b/...t/src/yamlRestTest/resources/rest-api-spec/test/entsearch/400_connector_sync_job_post.yml
@@ -11,6 +11,14 @@ setup:
           language: de
           is_native: false
           service_type: super-connector
+  - do:
+      connector.put:
+        connector_id: test-connector-detached-index
+        body:
+          name: my-connector
+          language: de
+          is_native: false
+          service_type: super-connector
 
 ---
 'Create connector sync job':
@@ -289,6 +297,16 @@ setup:
       catch: bad_request
 
 
+---
+'Create connector sync job with no index attached':
+  - do:
+      connector_sync_job.post:
+        body:
+          id: test-connector-detached-index
+          job_type: full
+          trigger_method: full
+      catch: bad_request
+
 ---
 "Create connector sync job fails for unprivileged user":
   - skip:

diff --git a/...ch/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/...ch/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
@@ -58,6 +58,7 @@
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -107,7 +108,6 @@ public void createConnectorWithDocId(PutConnectorAction.Request request, ActionL
         try {
             isDataIndexNameAlreadyInUse(indexName, connectorId, listener.delegateFailure((l, isIndexNameInUse) -> {
                 if (isIndexNameInUse) {
-
                     l.onFailure(
                         new ElasticsearchStatusException(
                             "Index name [" + indexName + "] is used by another connector.",
@@ -188,7 +188,7 @@ public void createConnectorWithAutoGeneratedId(
      * Creates a Connector with default values and specified parameters.
      *
      * @param description The description of the connector.
-     * @param indexName   The name of the index associated with the connector.
+     * @param indexName   The name of the index associated with the connector. It can be null to indicate that index is not attached yet.
      * @param isNative    Flag indicating if the connector is native; defaults to false if null.
      * @param language    The language supported by the connector.
      * @param name        The name of the connector; defaults to an empty string if null.
@@ -694,7 +694,8 @@ public void updateConnectorPipeline(UpdateConnectorPipelineAction.Request reques
     }
 
     /**
-     * Updates the index name property of a {@link Connector}.
+     * Updates the index name property of a {@link Connector}. Index name can be set to null to indicate that the connector
+     * is not associated with any index.
      *
      * @param request  The request for updating the connector's index name.
      * @param listener The listener for handling responses, including successful updates or errors.
@@ -720,8 +721,11 @@ public void updateConnectorIndexName(UpdateConnectorIndexNameAction.Request requ
                     new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
                         .id(connectorId)
                         .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-                        .source(Map.of(Connector.INDEX_NAME_FIELD.getPreferredName(), request.getIndexName()))
-
+                        .source(new HashMap<>() {
+                            {
+                                put(Connector.INDEX_NAME_FIELD.getPreferredName(), request.getIndexName());
+                            }
+                        })
                 );
                 client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> {
                     if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
@@ -912,6 +916,10 @@ private static ConnectorSearchResult hitToConnector(SearchHit searchHit) {
      * @param listener     The listener for handling boolean responses and errors.
      */
     private void isDataIndexNameAlreadyInUse(String indexName, String connectorId, ActionListener<Boolean> listener) {
+        if (indexName == null) {
+            listener.onResponse(false);
+            return;
+        }
         try {
             BoolQueryBuilder boolFilterQueryBuilder = new BoolQueryBuilder();
 

diff --git a/...ain/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java b/...ain/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java
@@ -8,13 +8,18 @@
 package org.elasticsearch.xpack.application.connector.action;
 
 import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 
 import java.io.IOException;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 /**
  * Abstract base class for action requests targeting the connectors index. Implements {@link org.elasticsearch.action.IndicesRequest}
  * to ensure index-level privilege support. This class defines the connectors index as the target for all derived action requests.
@@ -29,6 +34,24 @@ public ConnectorActionRequest(StreamInput in) throws IOException {
         super(in);
     }
 
+    /**
+     * Validates the given index name and updates the validation exception if the name is invalid.
+     *
+     * @param indexName The index name to validate. If null, no validation is performed.
+     * @param validationException The exception to accumulate validation errors.
+     * @return The updated or original {@code validationException} with any new validation errors added, if the index name is invalid.
+     */
+    public ActionRequestValidationException validateIndexName(String indexName, ActionRequestValidationException validationException) {
+        if (indexName != null) {
+            try {
+                MetadataCreateIndexService.validateIndexOrAliasName(indexName, InvalidIndexNameException::new);
+            } catch (InvalidIndexNameException e) {
+                return addValidationError(e.toString(), validationException);
+            }
+        }
+        return validationException;
+    }
+
     @Override
     public String[] indices() {
         return new String[] { ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN };

diff --git a/...c/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java b/...c/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java
@@ -11,14 +11,11 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
-import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -31,8 +28,6 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class PostConnectorAction {
@@ -46,6 +41,7 @@ public static class Request extends ConnectorActionRequest implements ToXContent
 
         @Nullable
         private final String description;
+        @Nullable
         private final String indexName;
         @Nullable
         private final Boolean isNative;
@@ -68,7 +64,7 @@ public Request(String description, String indexName, Boolean isNative, String la
         public Request(StreamInput in) throws IOException {
             super(in);
             this.description = in.readOptionalString();
-            this.indexName = in.readString();
+            this.indexName = in.readOptionalString();
             this.isNative = in.readOptionalBoolean();
             this.language = in.readOptionalString();
             this.name = in.readOptionalString();
@@ -90,7 +86,7 @@ public Request(StreamInput in) throws IOException {
 
         static {
             PARSER.declareString(optionalConstructorArg(), new ParseField("description"));
-            PARSER.declareString(constructorArg(), new ParseField("index_name"));
+            PARSER.declareStringOrNull(optionalConstructorArg(), new ParseField("index_name"));
             PARSER.declareBoolean(optionalConstructorArg(), new ParseField("is_native"));
             PARSER.declareString(optionalConstructorArg(), new ParseField("language"));
             PARSER.declareString(optionalConstructorArg(), new ParseField("name"));
@@ -116,7 +112,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 if (description != null) {
                     builder.field("description", description);
                 }
-                builder.field("index_name", indexName);
+                if (indexName != null) {
+                    builder.field("index_name", indexName);
+                }
                 if (isNative != null) {
                     builder.field("is_native", isNative);
                 }
@@ -138,14 +136,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
 
-            if (Strings.isNullOrEmpty(getIndexName())) {
-                validationException = addValidationError("[index_name] cannot be [null] or [\"\"]", validationException);
-            }
-            try {
-                MetadataCreateIndexService.validateIndexOrAliasName(getIndexName(), InvalidIndexNameException::new);
-            } catch (InvalidIndexNameException e) {
-                validationException = addValidationError(e.toString(), validationException);
-            }
+            validationException = validateIndexName(indexName, validationException);
 
             return validationException;
         }
@@ -154,7 +145,7 @@ public ActionRequestValidationException validate() {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeOptionalString(description);
-            out.writeString(indexName);
+            out.writeOptionalString(indexName);
             out.writeOptionalBoolean(isNative);
             out.writeOptionalString(language);
             out.writeOptionalString(name);

diff --git a/...rc/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java b/...rc/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java
@@ -13,14 +13,12 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -49,6 +47,7 @@ public static class Request extends ConnectorActionRequest implements IndicesReq
 
         @Nullable
         private final String description;
+        @Nullable
         private final String indexName;
         @Nullable
         private final Boolean isNative;
@@ -81,7 +80,7 @@ public Request(StreamInput in) throws IOException {
             super(in);
             this.connectorId = in.readString();
             this.description = in.readOptionalString();
-            this.indexName = in.readString();
+            this.indexName = in.readOptionalString();
             this.isNative = in.readOptionalBoolean();
             this.language = in.readOptionalString();
             this.name = in.readOptionalString();
@@ -104,7 +103,7 @@ public Request(StreamInput in) throws IOException {
 
         static {
             PARSER.declareString(optionalConstructorArg(), new ParseField("description"));
-            PARSER.declareString(optionalConstructorArg(), new ParseField("index_name"));
+            PARSER.declareStringOrNull(optionalConstructorArg(), new ParseField("index_name"));
             PARSER.declareBoolean(optionalConstructorArg(), new ParseField("is_native"));
             PARSER.declareString(optionalConstructorArg(), new ParseField("language"));
             PARSER.declareString(optionalConstructorArg(), new ParseField("name"));
@@ -130,7 +129,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 if (description != null) {
                     builder.field("description", description);
                 }
-                builder.field("index_name", indexName);
+                if (indexName != null) {
+                    builder.field("index_name", indexName);
+                }
                 if (isNative != null) {
                     builder.field("is_native", isNative);
                 }
@@ -156,14 +157,8 @@ public ActionRequestValidationException validate() {
             if (Strings.isNullOrEmpty(getConnectorId())) {
                 validationException = addValidationError("[connector_id] cannot be [null] or [\"\"]", validationException);
             }
-            if (Strings.isNullOrEmpty(getIndexName())) {
-                validationException = addValidationError("[index_name] cannot be [null] or [\"\"]", validationException);
-            }
-            try {
-                MetadataCreateIndexService.validateIndexOrAliasName(getIndexName(), InvalidIndexNameException::new);
-            } catch (InvalidIndexNameException e) {
-                validationException = addValidationError(e.toString(), validationException);
-            }
+
+            validationException = validateIndexName(indexName, validationException);
 
             return validationException;
         }
@@ -173,7 +168,7 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(connectorId);
             out.writeOptionalString(description);
-            out.writeString(indexName);
+            out.writeOptionalString(indexName);
             out.writeOptionalBoolean(isNative);
             out.writeOptionalString(language);
             out.writeOptionalString(name);