From 860ccfda72375839ab0b53db6913c60bf3d021ef Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 26 May 2023 17:31:50 -0500 Subject: [PATCH 1/3] Ignoring DataLifecycleServiceIT.testAutomaticForceMerge (#96395) Ignoring a failing test. Relates to #96084 --- .../java/org/elasticsearch/dlm/DataLifecycleServiceIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/dlm/src/internalClusterTest/java/org/elasticsearch/dlm/DataLifecycleServiceIT.java b/modules/dlm/src/internalClusterTest/java/org/elasticsearch/dlm/DataLifecycleServiceIT.java index 44849a1ead3f4..5b9632bf69277 100644 --- a/modules/dlm/src/internalClusterTest/java/org/elasticsearch/dlm/DataLifecycleServiceIT.java +++ b/modules/dlm/src/internalClusterTest/java/org/elasticsearch/dlm/DataLifecycleServiceIT.java @@ -259,6 +259,7 @@ public void testUpdatingLifecycleAppliesToAllBackingIndices() throws Exception { }); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/96084") public void testAutomaticForceMerge() throws Exception { /* * This test makes sure that (1) DLM does _not_ call forcemerge on an index in the same DLM pass when it rolls over the index and From a71e98863c970e54020b27a14a1d6444e0c17291 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Sat, 27 May 2023 09:37:30 +1000 Subject: [PATCH 2/3] [Test] Include DLM actions only when feature flag is enabled (#96118) This PR adds DLM feature flag checks when preparing the list of non-operator actions to ensure DLM actions are included only when the feature flag is enabled. Resolves: #96105 --- .../xpack/security/operator/Constants.java | 12 +++++++----- .../security/operator/OperatorPrivilegesIT.java | 1 - 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index dd60cabb63901..ab4172e6c1868 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.operator; +import org.elasticsearch.cluster.metadata.DataLifecycle; import org.elasticsearch.transport.TcpTransport; import java.util.Objects; @@ -108,7 +109,6 @@ public class Constants { "cluster:admin/xpack/application/search_application/list", "cluster:admin/xpack/application/search_application/put", "cluster:admin/xpack/application/search_application/render_query", - "cluster:admin/xpack/application/search_application/search", "cluster:admin/xpack/ccr/auto_follow_pattern/activate", "cluster:admin/xpack/ccr/auto_follow_pattern/delete", "cluster:admin/xpack/ccr/auto_follow_pattern/get", @@ -289,6 +289,7 @@ public class Constants { "cluster:monitor/nodes/info", "cluster:monitor/nodes/stats", "cluster:monitor/nodes/usage", + "cluster:monitor/profiling/status/get", "cluster:monitor/remote/info", "cluster:monitor/settings", "cluster:monitor/state", @@ -410,10 +411,10 @@ public class Constants { "indices:admin/data_stream/migrate", "indices:admin/data_stream/modify", "indices:admin/data_stream/promote", - "indices:admin/dlm/delete", - "indices:admin/dlm/get", - "indices:admin/dlm/put", - "indices:admin/dlm/explain", + DataLifecycle.isEnabled() ? "indices:admin/dlm/delete" : null, + DataLifecycle.isEnabled() ? "indices:admin/dlm/get" : null, + DataLifecycle.isEnabled() ? "indices:admin/dlm/put" : null, + DataLifecycle.isEnabled() ? "indices:admin/dlm/explain" : null, "indices:admin/delete", "indices:admin/flush", "indices:admin/flush[s]", @@ -485,6 +486,7 @@ public class Constants { "indices:data/read/sql/translate", "indices:data/read/sql/async/get", // org.elasticsearch.xpack.core.sql.SqlAsyncActionNames.SQL_ASYNC_GET_RESULT_ACTION_NAME "indices:data/read/tv", + "indices:data/read/xpack/application/search_application/search", "indices:data/read/xpack/ccr/shard_changes", "indices:data/read/xpack/enrich/coordinate_lookups", "indices:data/read/xpack/graph/explore", diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java index e941083679cfc..3870438c35bac 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java @@ -94,7 +94,6 @@ public void testOperatorUserCanCallNonOperatorOnlyApi() throws IOException { client().performRequest(mainRequest); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/96105") @SuppressWarnings("unchecked") public void testEveryActionIsEitherOperatorOnlyOrNonOperator() throws IOException { final String message = "An action should be declared to be either operator-only in [" From e5e2a0400f0f2b763bc8fc0da5e6b88498553744 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Sat, 27 May 2023 09:46:02 +1000 Subject: [PATCH 3/3] REST spec and YAML tests for cross-cluster API key APIs (#96339) This PR adds REST spec files and YAML tests for the new create and update cross-cluster API key APIs. Relates: #95714, #96085 --- ...security.create_cross_cluster_api_key.json | 34 ++ ...security.update_cross_cluster_api_key.json | 39 ++ x-pack/docs/en/rest-api/security.asciidoc | 29 ++ .../create-cross-cluster-api-key.asciidoc | 9 + .../update-cross-cluster-api-key.asciidoc | 9 + x-pack/plugin/build.gradle | 1 + .../test/api_key/50_cross_cluster.yml | 343 ++++++++++++++++++ 7 files changed, 464 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/security.create_cross_cluster_api_key.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/security.update_cross_cluster_api_key.json create mode 100644 x-pack/docs/en/rest-api/security/create-cross-cluster-api-key.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/update-cross-cluster-api-key.asciidoc create mode 100644 x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/50_cross_cluster.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.create_cross_cluster_api_key.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.create_cross_cluster_api_key.json new file mode 100644 index 0000000000000..98069a46e907a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.create_cross_cluster_api_key.json @@ -0,0 +1,34 @@ +{ + "security.create_cross_cluster_api_key": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html", + "description": "Creates a cross-cluster API key for API key based remote cluster access." + }, + "stability": "experimental", + "visibility": "feature_flag", + "feature_flag": "es.untrusted_remote_cluster_feature_flag_registered", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_security/cross_cluster/api_key", + "methods": [ + "POST" + ] + } + ] + }, + "params": {}, + "body": { + "description": "The request to create a cross-cluster API key", + "required": true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.update_cross_cluster_api_key.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.update_cross_cluster_api_key.json new file mode 100644 index 0000000000000..c0466bac79b91 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.update_cross_cluster_api_key.json @@ -0,0 +1,39 @@ +{ + "security.update_cross_cluster_api_key": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-update-cross-cluster-api-key.html", + "description": "Updates attributes of an existing cross-cluster API key." + }, + "stability": "experimental", + "visibility": "feature_flag", + "feature_flag": "es.untrusted_remote_cluster_feature_flag_registered", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_security/cross_cluster/api_key/{id}", + "methods": [ + "PUT" + ], + "parts": { + "id": { + "type": "string", + "description": "The ID of the cross-cluster API key to update" + } + } + } + ] + }, + "body": { + "description": "The request to update attributes of a cross-cluster API key.", + "required": true + } + } +} diff --git a/x-pack/docs/en/rest-api/security.asciidoc b/x-pack/docs/en/rest-api/security.asciidoc index a47f03cca1d2d..c0f30bb957cfb 100644 --- a/x-pack/docs/en/rest-api/security.asciidoc +++ b/x-pack/docs/en/rest-api/security.asciidoc @@ -61,6 +61,7 @@ without requiring basic authentication: [[security-api-keys]] === API Keys +ifeval::["{release-state}"=="released"] Use the following APIs to create, retrieve and invalidate API keys for access without requiring basic authentication: @@ -72,6 +73,30 @@ without requiring basic authentication: * <> * <> * <> +endif::[] + +ifeval::["{release-state}"!="released"] +Use the following APIs to create and update API keys for access via the REST interface +without requiring basic authentication: + +* <> +* <> +* <> +* <> + +Use the following APIs to create and update cross-cluster API keys for +API key based remote cluster access: + +* <> +* <> + +Use the following APIs to retrieve and invalidate API keys of all types: + +* <> +* <> +* <> +* <> +endif::[] [discrete] [[security-user-apis]] @@ -206,3 +231,7 @@ include::security/get-user-profile.asciidoc[] include::security/suggest-user-profile.asciidoc[] include::security/update-user-profile-data.asciidoc[] include::security/has-privileges-user-profile.asciidoc[] +ifeval::["{release-state}"!="released"] +include::security/create-cross-cluster-api-key.asciidoc[] +include::security/update-cross-cluster-api-key.asciidoc[] +endif::[] diff --git a/x-pack/docs/en/rest-api/security/create-cross-cluster-api-key.asciidoc b/x-pack/docs/en/rest-api/security/create-cross-cluster-api-key.asciidoc new file mode 100644 index 0000000000000..f655346a305e2 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/create-cross-cluster-api-key.asciidoc @@ -0,0 +1,9 @@ +[role="xpack"] +[[security-api-create-cross-cluster-api-key]] +=== Create Cross-Cluster API key API + +++++ +Create Cross-Cluster API key +++++ + +TODO: Placeholder diff --git a/x-pack/docs/en/rest-api/security/update-cross-cluster-api-key.asciidoc b/x-pack/docs/en/rest-api/security/update-cross-cluster-api-key.asciidoc new file mode 100644 index 0000000000000..2653c64069b35 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/update-cross-cluster-api-key.asciidoc @@ -0,0 +1,9 @@ +[role="xpack"] +[[security-api-update-cross-cluster-api-key]] +=== Update Cross-Cluster API key API + +++++ +Update Cross-Cluster API key +++++ + +TODO: Placeholder diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 93aa30ec68f44..51c1099a31612 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -70,6 +70,7 @@ if (BuildParams.isSnapshotBuild() == false) { // cross_cluster_search privilege is only available when untrusted_remote_cluster_feature_flag_registered is enabled // which requires snapshot build restTestBlacklist.add('privileges/11_builtin/Test get builtin privileges') + restTestBlacklist.add('api_key/50_cross_cluster/*') } tasks.named("yamlRestTest").configure { diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/50_cross_cluster.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/50_cross_cluster.yml new file mode 100644 index 0000000000000..6b1f5bc764151 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/50_cross_cluster.yml @@ -0,0 +1,343 @@ +--- +setup: + - skip: + features: headers + + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_role: + name: "admin_role" + body: > + { + "cluster": ["manage_security"] + } + + - do: + security.put_user: + username: "admin_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "admin_role" ], + "full_name" : "Admin user" + } + +--- +teardown: + - do: + security.delete_role: + name: "admin_role" + ignore: 404 + + - do: + security.delete_user: + username: "admin_user" + ignore: 404 + +--- +"Test create a cross-cluster API key": + - skip: + features: transform_and_set + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.create_cross_cluster_api_key: + body: > + { + "name": "my-cc-api-key", + "expiration": "1d", + "access": { + "search": [ + { + "names": ["logs*"], + "query": { + "term": { "category": "shared" } + }, + "field_security": { + "grant": ["*"], + "except": ["private"] + } + } + ], + "replication": [ + { + "names": ["archive"], + "allow_restricted_indices": false + } + ] + }, + "metadata": { + "answer": 42, + "tag": "dev" + } + } + - match: { name: "my-cc-api-key" } + - is_true: id + - is_true: api_key + - is_true: expiration + - set: { id: api_key_id } + - transform_and_set: { login_creds: "#base64EncodeCredentials(id,api_key)" } + - match: { encoded: $login_creds } + + # Authenticate with it via the REST interface should fail + - do: + catch: unauthorized + headers: + Authorization: ApiKey ${login_creds} + security.authenticate: { } + + - match: { "error.type": "security_exception" } + - match: + "error.reason": "authentication expected API key type of [rest], but API key [${api_key_id}] has type [cross_cluster]" + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.get_api_key: + id: "$api_key_id" + with_limited_by: true + + - length: { "api_keys": 1 } + - match: { "api_keys.0.id": "$api_key_id" } + - match: { "api_keys.0.name": "my-cc-api-key" } + - match: { "api_keys.0.type": "cross_cluster" } + - is_false: api_keys.0.invalidated + - match: { "api_keys.0.metadata": { "answer": 42, "tag": "dev" } } + - match: { "api_keys.0.role_descriptors": { + "cross_cluster": { + "cluster": [ + "cross_cluster_search", + "cross_cluster_replication" + ], + "indices": [ + { + "names": [ + "logs*" + ], + "privileges": [ + "read", + "read_cross_cluster", + "view_index_metadata" + ], + "field_security": { + "grant": [ + "*" + ], + "except": [ + "private" + ] + }, + "query": "{\"term\":{\"category\":\"shared\"}}", + "allow_restricted_indices": false + }, + { + "names": [ + "archive" + ], + "privileges": [ + "cross_cluster_replication", + "cross_cluster_replication_internal" + ], + "allow_restricted_indices": false + } + ], + "applications": [ ], + "run_as": [ ], + "metadata": { }, + "transient_metadata": { + "enabled": true + } + } + } + } + - is_false: api_keys.0.limited_by + +--- +"Test update a cross-cluster API Key": + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.create_cross_cluster_api_key: + body: > + { + "name": "my-cc-api-key", + "access": { + "search": [ + { + "names": ["logs*"] + } + ] + }, + "metadata": { "tag": "dev" } + } + - is_true: id + - set: { id: api_key_id } + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.update_cross_cluster_api_key: + id: "$api_key_id" + body: > + { + "access": { + "replication": [ + { + "names": ["archive"] + } + ] + }, + "metadata": { "tag": "prod" } + } + - match: { updated: true } + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.get_api_key: + id: "$api_key_id" + with_limited_by: true + + - length: { "api_keys": 1 } + - match: { "api_keys.0.id": "$api_key_id" } + - match: { "api_keys.0.name": "my-cc-api-key" } + - match: { "api_keys.0.type": "cross_cluster" } + - is_false: api_keys.0.invalidated + - match: { "api_keys.0.metadata": { "tag": "prod" } } + - match: { "api_keys.0.role_descriptors": { + "cross_cluster": { + "cluster": [ + "cross_cluster_replication" + ], + "indices": [ + { + "names": [ + "archive" + ], + "privileges": [ + "cross_cluster_replication", + "cross_cluster_replication_internal" + ], + "allow_restricted_indices": false + } + ], + "applications": [ ], + "run_as": [ ], + "metadata": { }, + "transient_metadata": { + "enabled": true + } + } + } + } + - is_false: api_keys.0.limited_by + + # No-op update + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.update_cross_cluster_api_key: + id: "$api_key_id" + body: > + { + "access": { + "replication": [ + { + "names": ["archive"] + } + ] + }, + "metadata": { "tag": "prod" } + } + - match: { updated: false } + +--- +"Test invalidate a cross-cluster API Key": + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.create_cross_cluster_api_key: + body: > + { + "name": "my-cc-api-key", + "access": { + "search": [ + { + "names": ["*"], + "allow_restricted_indices": true + } + ] + } + } + - is_true: id + - set: { id: api_key_id } + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.get_api_key: + id: "$api_key_id" + + - length: { "api_keys": 1 } + - match: { "api_keys.0.id": "$api_key_id" } + - match: { "api_keys.0.name": "my-cc-api-key" } + - match: { "api_keys.0.type": "cross_cluster" } + - is_false: api_keys.0.invalidated + - match: { "api_keys.0.metadata": { } } + - match: { "api_keys.0.role_descriptors": { + "cross_cluster": { + "cluster": [ + "cross_cluster_search" + ], + "indices": [ + { + "names": [ + "*" + ], + "privileges": [ + "read", + "read_cross_cluster", + "view_index_metadata" + ], + "allow_restricted_indices": true + } + ], + "applications": [ ], + "run_as": [ ], + "metadata": { }, + "transient_metadata": { + "enabled": true + } + } + } + } + - is_false: api_keys.0.limited_by + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.invalidate_api_key: + body: > + { + "ids": [ "${api_key_id}" ] + } + - length: { "invalidated_api_keys": 1 } + - match: { "invalidated_api_keys.0": "${api_key_id}" } + - length: { "previously_invalidated_api_keys": 0 } + - match: { "error_count": 0 } + + - do: + headers: + Authorization: "Basic YWRtaW5fdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # admin_user + security.get_api_key: + id: "$api_key_id" + + - length: { "api_keys": 1 } + - match: { "api_keys.0.id": "$api_key_id" } + - is_true: api_keys.0.invalidated