[Enterprise Search] Add .connector-secrets system index (#104766)

- Introduce new internal system index called .connector-secrets
- Add GET and POST requests for connector secrets
- Add permission sets for read and write connector secrets

docs/reference/rest-api/security/get-builtin-privileges.asciidoc

@@ -108,12 +108,14 @@ A successful call returns an object with "cluster" and "index" fields.
     "none",
     "post_behavioral_analytics_event",
     "read_ccr",
+    "read_connector_secrets",
     "read_fleet_secrets",
     "read_ilm",
     "read_pipeline",
     "read_security",
     "read_slm",
     "transport_client",
+    "write_connector_secrets",
     "write_fleet_secrets"
   ],
   "index" : [

rest-api-spec/src/main/resources/rest-api-spec/api/connector_secret.get.json

@@ -0,0 +1,28 @@
+{
+  "connector_secret.get": {
+    "documentation": {
+      "url": null,
+      "description": "Retrieves a secret stored by Connectors."
+    },
+    "stability": "experimental",
+    "visibility":"private",
+    "headers":{
+      "accept": [ "application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_connector/_secret/{id}",
+          "methods":[ "GET" ],
+          "parts":{
+            "id":{
+              "type":"string",
+              "description":"The ID of the secret"
+            }
+          }
+        }
+      ]
+    },
+    "params":{}
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/connector_secret.post.json

@@ -0,0 +1,26 @@
+{
+  "connector_secret.post": {
+    "documentation": {
+      "url": null,
+      "description": "Creates a secret for a Connector."
+    },
+    "stability": "experimental",
+    "visibility":"private",
+    "headers":{
+      "accept": [ "application/json" ]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_connector/_secret",
+          "methods":[ "POST" ]
+        }
+      ]
+    },
+    "params":{},
+    "body": {
+      "description":"The secret value to store",
+      "required":true
+    }
+  }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java

@@ -326,6 +326,16 @@ public class ClusterPrivilegeResolver {
         CROSS_CLUSTER_REPLICATION_PATTERN
     );
 
+    public static final NamedClusterPrivilege READ_CONNECTOR_SECRETS = new ActionClusterPrivilege(
+        "read_connector_secrets",
+        Set.of("cluster:admin/xpack/connector/secret/get")
+    );
+
+    public static final NamedClusterPrivilege WRITE_CONNECTOR_SECRETS = new ActionClusterPrivilege(
+        "write_connector_secrets",
+        Set.of("cluster:admin/xpack/connector/secret/post")
+    );
+
     private static final Map<String, NamedClusterPrivilege> VALUES = sortByAccessLevel(
         Stream.of(
             NONE,
@@ -380,7 +390,9 @@ public class ClusterPrivilegeResolver {
             POST_BEHAVIORAL_ANALYTICS_EVENT,
             MANAGE_SEARCH_QUERY_RULES,
             CROSS_CLUSTER_SEARCH,
-            CROSS_CLUSTER_REPLICATION
+            CROSS_CLUSTER_REPLICATION,
+            READ_CONNECTOR_SECRETS,
+            WRITE_CONNECTOR_SECRETS
         ).filter(Objects::nonNull).toList()
     );
 

x-pack/plugin/core/template-resources/src/main/resources/connector-secrets.json

@@ -0,0 +1,26 @@
+{
+  "settings": {
+    "index": {
+      "auto_expand_replicas": "0-1",
+      "number_of_shards": 1,
+      "number_of_replicas": 0,
+      "priority": 100,
+      "refresh_interval": "1s"
+    }
+  },
+  "mappings": {
+    "_doc" : {
+      "dynamic": false,
+      "_meta": {
+        "version": "${connector-secrets.version}",
+        "managed_index_mappings_version": ${connector-secrets.managed.index.version}
+      },
+      "properties": {
+        "value": {
+          "type": "keyword",
+          "index": false
+        }
+      }
+    }
+  }
+}

x-pack/plugin/ent-search/build.gradle

@@ -38,6 +38,13 @@ dependencies {
   module ':modules:search-business-rules'
 }
 
+testClusters.configureEach {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.autoconfiguration.enabled', 'false'
+  user username: 'x_pack_rest_user', password: 'x-pack-test-password'
+}
+
 tasks.named("dependencyLicenses").configure {
   mapping from: /jackson.*/, to: 'jackson'
 }

x-pack/plugin/ent-search/qa/rest/roles.yml

@@ -16,6 +16,8 @@ user:
   cluster:
     - post_behavioral_analytics_event
     - manage_api_key
+    - read_connector_secrets
+    - write_connector_secrets
   indices:
     - names: [
       "test-index1",

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/500_connector_secret_post.yml

@@ -0,0 +1,55 @@
+setup:
+  - skip:
+      version: " - 8.12.99"
+      reason: Introduced in 8.13.0
+
+---
+'Post connector secret - admin':
+  - do:
+      connector_secret.post:
+        body:
+          value: my-secret
+  - set: { id: id }
+  - match: { id: $id }
+  - do:
+      connector_secret.get:
+        id: $id
+  - match: { value: my-secret }
+
+---
+'Post connector secret - authorized user':
+  - skip:
+      features: headers
+
+  - do:
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" }  # user
+      connector_secret.post:
+        body:
+          value: my-secret
+  - set: { id: id }
+  - match: { id: $id }
+  - do:
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" }  # user
+      connector_secret.get:
+        id: $id
+  - match: { value: my-secret }
+
+---
+'Post connector secret - unauthorized user':
+  - skip:
+      features: headers
+
+  - do:
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVucHJpdmlsZWdlZDplbnRzZWFyY2gtdW5wcml2aWxlZ2VkLXVzZXI=" }  # unprivileged
+      connector_secret.post:
+        body:
+          value: my-secret
+      catch: unauthorized
+
+---
+'Post connector secret when id is missing should fail':
+  - do:
+      connector_secret.post:
+        body:
+          value: null
+      catch: bad_request

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/510_connector_secret_get.yml

@@ -0,0 +1,60 @@
+setup:
+  - skip:
+      version: " - 8.12.99"
+      reason: Introduced in 8.13.0
+
+---
+'Get connector secret - admin':
+  - do:
+      connector_secret.post:
+        body:
+          value: my-secret
+  - set: { id: id }
+  - match: { id: $id }
+  - do:
+      connector_secret.get:
+        id: $id
+  - match: { value: my-secret }
+
+---
+'Get connector secret - user with privileges':
+  - skip:
+      features: headers
+
+  - do:
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" }  # user
+      connector_secret.post:
+        body:
+          value: my-secret
+  - set: { id: id }
+  - match: { id: $id }
+  - do:
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" }  # user
+      connector_secret.get:
+        id: $id
+  - match: { value: my-secret }
+
+---
+'Get connector secret - user without privileges':
+  - skip:
+      features: headers
+
+  - do:
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" }  # user
+      connector_secret.post:
+        body:
+          value: my-secret
+  - set: { id: id }
+  - match: { id: $id }
+  - do:
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVucHJpdmlsZWdlZDplbnRzZWFyY2gtdW5wcml2aWxlZ2VkLXVzZXI=" }  # unprivileged
+      connector_secret.get:
+        id: $id
+      catch: unauthorized
+
+---
+'Get connector secret - Missing secret id':
+  - do:
+      connector_secret.get:
+        id: non-existing-secret-id
+      catch: missing

x-pack/plugin/ent-search/src/javaRestTest/java/org/elasticsearch/xpack/entsearch/ConnectorSecretsSystemIndexIT.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.entsearch;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class ConnectorSecretsSystemIndexIT extends ESRestTestCase {
+
+    static final String BASIC_AUTH_VALUE = basicAuthHeaderValue(
+        "x_pack_rest_user",
+        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING
+    );
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build();
+    }
+
+    public void testConnectorSecretsCRUD() throws Exception {
+        // post secret
+        final String secretJson = getPostSecretJson();
+        Request postRequest = new Request("POST", "/_connector/_secret/");
+        postRequest.setJsonEntity(secretJson);
+        Response postResponse = client().performRequest(postRequest);
+        assertThat(postResponse.getStatusLine().getStatusCode(), is(200));
+        Map<String, Object> responseMap = getResponseMap(postResponse);
+        assertThat(responseMap.size(), is(1));
+        assertTrue(responseMap.containsKey("id"));
+        final String id = responseMap.get("id").toString();
+
+        // get secret
+        Request getRequest = new Request("GET", "/_connector/_secret/" + id);
+        Response getResponse = client().performRequest(getRequest);
+        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
+        responseMap = getResponseMap(getResponse);
+        assertThat(responseMap.size(), is(2));
+        assertTrue(responseMap.containsKey("id"));
+        assertTrue(responseMap.containsKey("value"));
+        assertThat(responseMap.get("value"), is("test secret"));
+    }
+
+    public void testPostInvalidSecretBody() throws Exception {
+        Request postRequest = new Request("POST", "/_connector/_secret/");
+        postRequest.setJsonEntity("""
+            {"something":"else"}""");
+        ResponseException re = expectThrows(ResponseException.class, () -> client().performRequest(postRequest));
+        Response getResponse = re.getResponse();
+        assertThat(getResponse.getStatusLine().getStatusCode(), is(400));
+    }
+
+    public void testGetNonExistingSecret() {
+        Request getRequest = new Request("GET", "/_connector/_secret/123");
+        ResponseException re = expectThrows(ResponseException.class, () -> client().performRequest(getRequest));
+        Response getResponse = re.getResponse();
+        assertThat(getResponse.getStatusLine().getStatusCode(), is(404));
+    }
+
+    private String getPostSecretJson() throws IOException {
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            {
+                builder.field("value", "test secret");
+            }
+            builder.endObject();
+            return BytesReference.bytes(builder).utf8ToString();
+        }
+    }
+
+    private Map<String, Object> getResponseMap(Response response) throws IOException {
+        return XContentHelper.convertToMap(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity()), false);
+    }
+}

x-pack/plugin/ent-search/src/main/java/module-info.java

@@ -39,4 +39,6 @@
     exports org.elasticsearch.xpack.application.connector.syncjob.action;
 
     provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.application.EnterpriseSearchFeatures;
+
+    exports org.elasticsearch.xpack.application.connector.secrets.action to org.elasticsearch.server;
 }