From 2b0f455496c83e322899e7ff3b491c527a232de9 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Wed, 6 Aug 2025 10:03:53 -0400 Subject: [PATCH 01/25] added privilege and test --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 5 +++++ .../core/security/authz/store/ReservedRolesStoreTests.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index eb0c77deb86c2..aa09791067334 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -174,6 +174,11 @@ static RoleDescriptor kibanaSystem(String name) { .privileges("write", "delete", "create_index") .allowRestrictedIndices(true) .build(), + // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. + RoleDescriptor.IndicesPrivileges.builder() + .indices(".integration_knowledge*") + .privileges("all") + .build(), // Other Fleet indices. Kibana reads and writes to these indices to manage // Elastic Agents. RoleDescriptor.IndicesPrivileges.builder() diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 04e0355f7a773..56381ec171b4c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -936,6 +936,11 @@ public void testKibanaSystemRole() { ".fleet-fileds" ).forEach(index -> assertAllIndicesAccessAllowed(kibanaRole, index)); + // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. + Arrays.asList(".integration_knowledge" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach( + index -> assertAllIndicesAccessAllowed(kibanaRole, index) + ); + final IndexAbstraction dotFleetSecretsIndex = mockIndexAbstraction(".fleet-secrets"); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(dotFleetSecretsIndex), is(false)); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(dotFleetSecretsIndex), is(false)); From 774c20bab17bd45c93886bdda6c3eb38d348e99a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 6 Aug 2025 14:20:59 +0000 Subject: [PATCH 02/25] [CI] Auto commit changes from spotless --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 8 +++----- .../security/authz/store/ReservedRolesStoreTests.java | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 020b283ec2fc8..6c1ab6ba189f3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -174,11 +174,9 @@ static RoleDescriptor kibanaSystem(String name) { .privileges("write", "delete", "create_index") .allowRestrictedIndices(true) .build(), - // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. - RoleDescriptor.IndicesPrivileges.builder() - .indices(".integration_knowledge*") - .privileges("all") - .build(), + // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI + // assistants. + RoleDescriptor.IndicesPrivileges.builder().indices(".integration_knowledge*").privileges("all").build(), // Other Fleet indices. Kibana reads and writes to these indices to manage // Elastic Agents. RoleDescriptor.IndicesPrivileges.builder() diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 581e53b7d6eb5..d5ece95e430ed 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -937,9 +937,8 @@ public void testKibanaSystemRole() { ).forEach(index -> assertAllIndicesAccessAllowed(kibanaRole, index)); // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. - Arrays.asList(".integration_knowledge" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach( - index -> assertAllIndicesAccessAllowed(kibanaRole, index) - ); + Arrays.asList(".integration_knowledge" + randomAlphaOfLength(randomIntBetween(0, 13))) + .forEach(index -> assertAllIndicesAccessAllowed(kibanaRole, index)); final IndexAbstraction dotFleetSecretsIndex = mockIndexAbstraction(".fleet-secrets"); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(dotFleetSecretsIndex), is(false)); From 5e65044e5f7574527dcbb55ac0c4b26b791fb95c Mon Sep 17 00:00:00 2001 From: Supplementing Date: Wed, 6 Aug 2025 11:25:02 -0400 Subject: [PATCH 03/25] changed privileges --- .../KibanaOwnedReservedRoleDescriptors.java | 2 +- .../authz/store/ReservedRolesStoreTests.java | 26 ++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index aa09791067334..704cbb697a5ed 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -177,7 +177,7 @@ static RoleDescriptor kibanaSystem(String name) { // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. RoleDescriptor.IndicesPrivileges.builder() .indices(".integration_knowledge*") - .privileges("all") + .privileges("read", "write", "create_index") .build(), // Other Fleet indices. Kibana reads and writes to these indices to manage // Elastic Agents. diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 56381ec171b4c..f859cfa177b2c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -937,9 +937,29 @@ public void testKibanaSystemRole() { ).forEach(index -> assertAllIndicesAccessAllowed(kibanaRole, index)); // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. - Arrays.asList(".integration_knowledge" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach( - index -> assertAllIndicesAccessAllowed(kibanaRole, index) - ); + Arrays.asList(".integration_knowledge" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((index) -> { + final IndexAbstraction indexAbstraction = mockIndexAbstraction(index); + assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false)); + assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(indexAbstraction), is(false)); + assertThat( + kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), + is(false) + ); + assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportMultiSearchAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(indexAbstraction), is(false)); + assertThat( + kibanaRole.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(indexAbstraction), + is(false) + ); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(false)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(false)); + }); final IndexAbstraction dotFleetSecretsIndex = mockIndexAbstraction(".fleet-secrets"); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(dotFleetSecretsIndex), is(false)); From b152bc9cebd5d63fdb8098a2eba389fcc3ea098b Mon Sep 17 00:00:00 2001 From: Supplementing Date: Wed, 6 Aug 2025 11:29:55 -0400 Subject: [PATCH 04/25] removed conflict i missed --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 70223675ba818..ec2374705bac5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -174,17 +174,11 @@ static RoleDescriptor kibanaSystem(String name) { .privileges("write", "delete", "create_index") .allowRestrictedIndices(true) .build(), -<<<<<<< HEAD // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. RoleDescriptor.IndicesPrivileges.builder() .indices(".integration_knowledge*") .privileges("read", "write", "create_index") .build(), -======= - // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI - // assistants. - RoleDescriptor.IndicesPrivileges.builder().indices(".integration_knowledge*").privileges("all").build(), ->>>>>>> 774c20bab17bd45c93886bdda6c3eb38d348e99a // Other Fleet indices. Kibana reads and writes to these indices to manage // Elastic Agents. RoleDescriptor.IndicesPrivileges.builder() From 2b42f6465f08c6b04dca77a0609513d20ba58f07 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 6 Aug 2025 15:42:18 +0000 Subject: [PATCH 05/25] [CI] Auto commit changes from spotless --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index ec2374705bac5..922cc6659c12e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -174,7 +174,8 @@ static RoleDescriptor kibanaSystem(String name) { .privileges("write", "delete", "create_index") .allowRestrictedIndices(true) .build(), - // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. + // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI + // assistants. RoleDescriptor.IndicesPrivileges.builder() .indices(".integration_knowledge*") .privileges("read", "write", "create_index") From d2ba854db5c0223ad5eda57becf0b35099650d4e Mon Sep 17 00:00:00 2001 From: Supplementing Date: Thu, 7 Aug 2025 09:20:20 -0400 Subject: [PATCH 06/25] updated test --- .../core/security/authz/store/ReservedRolesStoreTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 47cf0e54b76e4..201fb9d110fe3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -957,7 +957,7 @@ public void testKibanaSystemRole() { kibanaRole.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(indexAbstraction), is(false) ); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(false)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(true)); assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(false)); }); From 67014a9df86508a5ff286feb78423743125c5327 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Thu, 7 Aug 2025 09:47:20 -0400 Subject: [PATCH 07/25] rolled back test to be correct to investigate failure --- .../core/security/authz/store/ReservedRolesStoreTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 201fb9d110fe3..47cf0e54b76e4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -957,7 +957,7 @@ public void testKibanaSystemRole() { kibanaRole.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(indexAbstraction), is(false) ); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(false)); assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(false)); }); From e054b3500362d0b286755c3915dc2d7bdbc9c5e6 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Fri, 8 Aug 2025 11:14:34 -0400 Subject: [PATCH 08/25] added additional indexAbstraction function with actual type to avoid adding the wrong privileges --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 1 + .../security/authz/store/ReservedRolesStoreTests.java | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 922cc6659c12e..0cad70efc18f9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -179,6 +179,7 @@ static RoleDescriptor kibanaSystem(String name) { RoleDescriptor.IndicesPrivileges.builder() .indices(".integration_knowledge*") .privileges("read", "write", "create_index") + .allowRestrictedIndices(true) .build(), // Other Fleet indices. Kibana reads and writes to these indices to manage // Elastic Agents. diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 47cf0e54b76e4..605cfc5f4f99c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -938,7 +938,7 @@ public void testKibanaSystemRole() { // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. Arrays.asList(".integration_knowledge" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((index) -> { - final IndexAbstraction indexAbstraction = mockIndexAbstraction(index); + final IndexAbstraction indexAbstraction = mockIndexAbstraction(index, IndexAbstraction.Type.CONCRETE_INDEX); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false)); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(indexAbstraction), is(false)); assertThat( @@ -4255,4 +4255,11 @@ private IndexAbstraction mockIndexAbstraction(String name) { ); return mock; } + + private IndexAbstraction mockIndexAbstraction(String name, IndexAbstraction.Type type) { + IndexAbstraction mock = mock(IndexAbstraction.class); + when(mock.getName()).thenReturn(name); + when(mock.getType()).thenReturn(type); + return mock; + } } From 1a6871d9f21b4cb347880cde1781f7de9f0ce134 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Fri, 8 Aug 2025 13:52:58 -0400 Subject: [PATCH 09/25] changed test to true for now due to BWC logic --- .../core/security/authz/store/ReservedRolesStoreTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 605cfc5f4f99c..cd4dd774c8f4d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -957,7 +957,8 @@ public void testKibanaSystemRole() { kibanaRole.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(indexAbstraction), is(false) ); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(false)); + // In the future, this PutMappingAction check will be false but the BWC check is giving the privilege for all indices with 'write' access + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(true)); assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(false)); }); From e0ea28f1b989be104963646207032ff6b46ae901 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 8 Aug 2025 17:59:12 +0000 Subject: [PATCH 10/25] [CI] Auto commit changes from spotless --- .../core/security/authz/store/ReservedRolesStoreTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 9954c8d4c4e78..358388d9f12a8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -957,7 +957,8 @@ public void testKibanaSystemRole() { kibanaRole.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(indexAbstraction), is(false) ); - // In the future, this PutMappingAction check will be false but the BWC check is giving the privilege for all indices with 'write' access + // In the future, this PutMappingAction check will be false but the BWC check is giving the privilege for all indices with + // 'write' access assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(true)); assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(false)); }); From 7ac998d616344e02975cab026bd1650379c7c6c4 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Tue, 12 Aug 2025 09:32:47 -0400 Subject: [PATCH 11/25] changed to let index management be the responsibility of ES --- .../authz/store/ReservedRolesStoreTests.java | 26 -------------- .../security/test/TestRestrictedIndices.java | 3 +- .../fleet-integration-knowledge.json | 34 +++++++++++++++++++ .../xpack/fleet/FleetSystemIndicesIT.java | 16 +++++++++ .../org/elasticsearch/xpack/fleet/Fleet.java | 20 ++++++++++- .../elasticsearch/xpack/fleet/FleetTests.java | 5 ++- 6 files changed, 75 insertions(+), 29 deletions(-) create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 9954c8d4c4e78..b76f99d728a84 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -936,32 +936,6 @@ public void testKibanaSystemRole() { ".fleet-fileds" ).forEach(index -> assertAllIndicesAccessAllowed(kibanaRole, index)); - // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants. - Arrays.asList(".integration_knowledge" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((index) -> { - final IndexAbstraction indexAbstraction = mockIndexAbstraction(index, IndexAbstraction.Type.CONCRETE_INDEX); - assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false)); - assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(indexAbstraction), is(false)); - assertThat( - kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), - is(false) - ); - assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true)); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), is(true)); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(indexAbstraction), is(true)); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(indexAbstraction), is(true)); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(indexAbstraction), is(true)); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportMultiSearchAction.TYPE.name()).test(indexAbstraction), is(true)); - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(indexAbstraction), is(true)); - assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(indexAbstraction), is(false)); - assertThat( - kibanaRole.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(indexAbstraction), - is(false) - ); - // In the future, this PutMappingAction check will be false but the BWC check is giving the privilege for all indices with 'write' access - assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(true)); - assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(false)); - }); - final IndexAbstraction dotFleetSecretsIndex = mockIndexAbstraction(".fleet-secrets"); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(dotFleetSecretsIndex), is(false)); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(dotFleetSecretsIndex), is(false)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java index 28fea1a7099ed..265a68bb2c9b8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java @@ -103,7 +103,8 @@ public class TestRestrictedIndices { SystemIndexDescriptorUtils.createUnmanaged(".fleet-policies-[0-9]+*", "fleet policies"), SystemIndexDescriptorUtils.createUnmanaged(".fleet-policies-leader*", "fleet policies leader"), SystemIndexDescriptorUtils.createUnmanaged(".fleet-servers*", "fleet servers"), - SystemIndexDescriptorUtils.createUnmanaged(".fleet-artifacts*", "fleet artifacts") + SystemIndexDescriptorUtils.createUnmanaged(".fleet-artifacts*", "fleet artifacts"), + SystemIndexDescriptorUtils.createUnmanaged(".integration_knowledge*", "integration knowledge") ), List.of( new SystemDataStreamDescriptor( diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json new file mode 100644 index 0000000000000..49ea8eb892c29 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json @@ -0,0 +1,34 @@ +{ + "settings": { + "auto_expand_replicas": "0-1", + "index.hidden": true + }, + "mappings": { + "_doc": { + "dynamic": false, + "_meta": { + "version": "${fleet.version}", + "managed_index_mappings_version": "${fleet.managed.index.version}", + "description": "Integration package knowledge base content storage", + "managed": true + }, + "properties": { + "filename": { + "type": "keyword" + }, + "content": { + "type": "semantic_text" + }, + "version": { + "type": "version" + }, + "package_name": { + "type": "keyword" + }, + "installed_at": { + "type": "date" + } + } + } + } +} diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index 99f04d2997608..054039073a395 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -293,4 +293,20 @@ public void verifyFileDeliveryDataILMPolicyExists() throws Exception { assertThat(policyMap.size(), equalTo(2)); }); } + + public void testCreationOfIntegrationKnowledge() throws Exception { + Request request = new Request("PUT", ".integration-knowledge"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + + request = new Request("GET", ".integration-knowledge/_mapping"); + response = client().performRequest(request); + String responseBody = EntityUtils.toString(response.getEntity()); + assertThat(responseBody, containsString("knowledge_content")); + + request = new Request("GET", ".integration-knowledge-7/_mapping"); + response = client().performRequest(request); + responseBody = EntityUtils.toString(response.getEntity()); + assertThat(responseBody, containsString("knowledge_content")); + } } diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index e21d13524f1bf..350bddb9702a6 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -87,6 +87,7 @@ public class Fleet extends Plugin implements SystemIndexPlugin { private static final int FLEET_SERVERS_MAPPINGS_VERSION = 1; private static final int FLEET_ARTIFACTS_MAPPINGS_VERSION = 1; private static final int FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION = 1; + private static final int FLEET_INTEGRATION_KNOWLEDGE_MAPPINGS_VERSION = 1; @Override public Collection createComponents(PluginServices services) { @@ -111,7 +112,8 @@ public Collection getSystemIndexDescriptors(Settings sett fleetPoliciesSystemIndexDescriptor(), fleetPoliciesLeaderSystemIndexDescriptor(), fleetServersSystemIndexDescriptors(), - fleetArtifactsSystemIndexDescriptors() + fleetArtifactsSystemIndexDescriptors(), + fleetIntegrationKnowledgeSystemIndexDescriptor() ); } @@ -267,6 +269,22 @@ private static SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { .build(); } + private static SystemIndexDescriptor fleetIntegrationKnowledgeSystemIndexDescriptor() { + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + request.source(loadTemplateSource("/fleet-integration-knowledge.json", FLEET_INTEGRATION_KNOWLEDGE_MAPPINGS_VERSION), XContentType.JSON); + + return SystemIndexDescriptor.builder() + .setType(Type.INTERNAL_MANAGED) + .setOrigin(FLEET_ORIGIN) + .setMappings(request.mappings()) + .setSettings(request.settings()) + .setPrimaryIndex(".integration_knowledge-" + CURRENT_INDEX_VERSION) + .setIndexPattern(".integration_knowledge*") + .setAliasName(".integration_knowledge") + .setDescription("Integration package knowledge base content storage") + .build(); + } + private static SystemDataStreamDescriptor fleetActionsResultsDescriptor() { final String source = loadTemplateSource("/fleet-actions-results.json", FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, source)) { diff --git a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java index e8b2738da603f..52559676c91d1 100644 --- a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java +++ b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java @@ -45,7 +45,8 @@ public void testFleetIndexNames() { ".fleet-policies-leader*", ".fleet-enrollment-api-keys*", ".fleet-artifacts*", - ".fleet-secrets*" + ".fleet-secrets*", + ".integration_knowledge*" ) ); @@ -60,6 +61,8 @@ public void testFleetIndexNames() { assertFalse(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-actions-results"))); assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-secrets"))); + + assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".integration_knowledge"))); } public void testFleetFeature() { From fa552d1d0f016960b7ab6975bddf630c80c0764b Mon Sep 17 00:00:00 2001 From: Mason Herron <46727170+Supplementing@users.noreply.github.com> Date: Tue, 12 Aug 2025 09:37:51 -0400 Subject: [PATCH 12/25] Removed unneeded function --- .../core/security/authz/store/ReservedRolesStoreTests.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index b76f99d728a84..65fd9f75911cb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -4306,10 +4306,4 @@ private IndexAbstraction mockIndexAbstraction(String name) { return mock; } - private IndexAbstraction mockIndexAbstraction(String name, IndexAbstraction.Type type) { - IndexAbstraction mock = mock(IndexAbstraction.class); - when(mock.getName()).thenReturn(name); - when(mock.getType()).thenReturn(type); - return mock; - } } From 1f87382bb11d2242335d448a1925642fc54ea1b4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 12 Aug 2025 13:44:10 +0000 Subject: [PATCH 13/25] [CI] Auto commit changes from spotless --- .../src/main/java/org/elasticsearch/xpack/fleet/Fleet.java | 5 ++++- .../test/java/org/elasticsearch/xpack/fleet/FleetTests.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 350bddb9702a6..7adb64933e718 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -271,7 +271,10 @@ private static SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { private static SystemIndexDescriptor fleetIntegrationKnowledgeSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-integration-knowledge.json", FLEET_INTEGRATION_KNOWLEDGE_MAPPINGS_VERSION), XContentType.JSON); + request.source( + loadTemplateSource("/fleet-integration-knowledge.json", FLEET_INTEGRATION_KNOWLEDGE_MAPPINGS_VERSION), + XContentType.JSON + ); return SystemIndexDescriptor.builder() .setType(Type.INTERNAL_MANAGED) diff --git a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java index 52559676c91d1..b8bd493769c67 100644 --- a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java +++ b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java @@ -61,7 +61,7 @@ public void testFleetIndexNames() { assertFalse(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-actions-results"))); assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-secrets"))); - + assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".integration_knowledge"))); } From d5f0fd80bbdeee0ad903ed44d6a9ec64ae2396ad Mon Sep 17 00:00:00 2001 From: Supplementing Date: Tue, 12 Aug 2025 11:10:09 -0400 Subject: [PATCH 14/25] fixed formatting of variable in json --- .../src/main/resources/fleet-integration-knowledge.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json index 49ea8eb892c29..24b5f5e815ee8 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json @@ -8,7 +8,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "managed_index_mappings_version": "${fleet.managed.index.version}", + "managed_index_mappings_version": ${fleet.managed.index.version}, "description": "Integration package knowledge base content storage", "managed": true }, From da9eec1bf4e41ee51d16bf78dc98c7f2cb15c07d Mon Sep 17 00:00:00 2001 From: Mason Herron <46727170+Supplementing@users.noreply.github.com> Date: Tue, 12 Aug 2025 15:32:32 -0400 Subject: [PATCH 15/25] Update x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java Co-authored-by: Jen Huang --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index f94327da04a81..42c0eecf43561 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -174,7 +174,7 @@ static RoleDescriptor kibanaSystem(String name) { .privileges("write", "delete", "create_index") .allowRestrictedIndices(true) .build(), - // Knowledge base. Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI + // Integrations knowledge base: Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants to support integrations // assistants. RoleDescriptor.IndicesPrivileges.builder() .indices(".integration_knowledge*") From b2ba08ca80ebb746f34bdc1533dcbd68236368f8 Mon Sep 17 00:00:00 2001 From: Mason Herron <46727170+Supplementing@users.noreply.github.com> Date: Tue, 12 Aug 2025 15:32:53 -0400 Subject: [PATCH 16/25] Update x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java Co-authored-by: Jen Huang --- .../xpack/core/security/test/TestRestrictedIndices.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java index 265a68bb2c9b8..efd94f0d7eade 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java @@ -104,7 +104,7 @@ public class TestRestrictedIndices { SystemIndexDescriptorUtils.createUnmanaged(".fleet-policies-leader*", "fleet policies leader"), SystemIndexDescriptorUtils.createUnmanaged(".fleet-servers*", "fleet servers"), SystemIndexDescriptorUtils.createUnmanaged(".fleet-artifacts*", "fleet artifacts"), - SystemIndexDescriptorUtils.createUnmanaged(".integration_knowledge*", "integration knowledge") + SystemIndexDescriptorUtils.createUnmanaged(".integration_knowledge*", "fleet integration knowledge base") ), List.of( new SystemDataStreamDescriptor( From 89e6ac6c51742a4fd17e76091728043617081367 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Tue, 12 Aug 2025 15:39:08 -0400 Subject: [PATCH 17/25] added changelog and removed whitespace --- docs/changelog/132506.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/132506.yaml diff --git a/docs/changelog/132506.yaml b/docs/changelog/132506.yaml new file mode 100644 index 0000000000000..f5580b89c06ec --- /dev/null +++ b/docs/changelog/132506.yaml @@ -0,0 +1,5 @@ +pr: 132506 +summary: Add .integration_knowledge system index for usage by AI assistants +area: Fleet +type: feature +issues: [] From 1c9a8eef221e5bb285947e38daffeca03510e081 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Tue, 12 Aug 2025 15:40:10 -0400 Subject: [PATCH 18/25] actually remove whitespace --- .../xpack/core/security/authz/store/ReservedRolesStoreTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 65fd9f75911cb..34dd684e83166 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -4305,5 +4305,4 @@ private IndexAbstraction mockIndexAbstraction(String name) { ); return mock; } - } From 2683a615e49ea02b704e5e686e460c6bfd71d285 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 12 Aug 2025 19:46:20 +0000 Subject: [PATCH 19/25] [CI] Auto commit changes from spotless --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 42c0eecf43561..ac128fdc7a41c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -174,7 +174,8 @@ static RoleDescriptor kibanaSystem(String name) { .privileges("write", "delete", "create_index") .allowRestrictedIndices(true) .build(), - // Integrations knowledge base: Fleet creates, manages, and uses this index to store knowledge base documents to be consumed by AI assistants to support integrations + // Integrations knowledge base: Fleet creates, manages, and uses this index to store knowledge base documents to be consumed + // by AI assistants to support integrations // assistants. RoleDescriptor.IndicesPrivileges.builder() .indices(".integration_knowledge*") From a725405ad4f33afb23fbbf7fd0ec6d3fc8bbe5b1 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Tue, 12 Aug 2025 16:15:00 -0400 Subject: [PATCH 20/25] changed area in changelog --- docs/changelog/132506.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/132506.yaml b/docs/changelog/132506.yaml index f5580b89c06ec..3fd90cb657b49 100644 --- a/docs/changelog/132506.yaml +++ b/docs/changelog/132506.yaml @@ -1,5 +1,5 @@ pr: 132506 summary: Add .integration_knowledge system index for usage by AI assistants -area: Fleet +area: Infra/Core type: feature issues: [] From 356c20c62750bef5a694f59196c4c90772f82b93 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Tue, 12 Aug 2025 16:47:00 -0400 Subject: [PATCH 21/25] fixed indicies IT to use correct url --- .../org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index 054039073a395..07c81d12d6227 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -295,16 +295,16 @@ public void verifyFileDeliveryDataILMPolicyExists() throws Exception { } public void testCreationOfIntegrationKnowledge() throws Exception { - Request request = new Request("PUT", ".integration-knowledge"); + Request request = new Request("PUT", ".integration_knowledge"); Response response = client().performRequest(request); assertEquals(200, response.getStatusLine().getStatusCode()); - request = new Request("GET", ".integration-knowledge/_mapping"); + request = new Request("GET", ".integration_knowledge/_mapping"); response = client().performRequest(request); String responseBody = EntityUtils.toString(response.getEntity()); assertThat(responseBody, containsString("knowledge_content")); - request = new Request("GET", ".integration-knowledge-7/_mapping"); + request = new Request("GET", ".integration_knowledge-7/_mapping"); response = client().performRequest(request); responseBody = EntityUtils.toString(response.getEntity()); assertThat(responseBody, containsString("knowledge_content")); From 945371605d2d128f0e66995ee8cc74d604d74102 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Fri, 15 Aug 2025 08:53:39 -0600 Subject: [PATCH 22/25] fixed mapping issues by updating plugins needed to use types, also fixed tests to assert on the correct field --- x-pack/plugin/fleet/build.gradle | 3 +++ .../org/elasticsearch/xpack/fleet/AbstractFleetIT.java | 2 ++ .../elasticsearch/xpack/fleet/FleetSystemIndicesIT.java | 8 ++++++-- .../java/org/elasticsearch/xpack/fleet/FleetRestIT.java | 2 ++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/fleet/build.gradle b/x-pack/plugin/fleet/build.gradle index 4b19ff849fad8..a9643d796dbf6 100644 --- a/x-pack/plugin/fleet/build.gradle +++ b/x-pack/plugin/fleet/build.gradle @@ -24,7 +24,10 @@ dependencies { javaRestTestImplementation(project(path: xpackModule('core'))) javaRestTestImplementation(testArtifact(project(xpackModule('core')))) compileOnly project(path: xpackModule('ilm')) + testImplementation project(path: xpackModule('mapper-version')) clusterModules project(xpackModule('ilm')) + clusterModules project(xpackModule('mapper-version')) + clusterModules project(xpackModule('inference')) clusterModules project(':modules:data-streams') } diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java index a835b9caa7a65..bae9c80ea0f60 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java @@ -17,6 +17,8 @@ public abstract class AbstractFleetIT extends ESRestTestCase { public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("x-pack-fleet") .module("x-pack-ilm") + .module("mapper-version") + .module("x-pack-inference") .module("data-streams") .setting("xpack.security.enabled", "true") .setting("xpack.security.autoconfiguration.enabled", "false") diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index 07c81d12d6227..21c7646ffb7e1 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -9,7 +9,9 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentHelper; @@ -300,13 +302,15 @@ public void testCreationOfIntegrationKnowledge() throws Exception { assertEquals(200, response.getStatusLine().getStatusCode()); request = new Request("GET", ".integration_knowledge/_mapping"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); response = client().performRequest(request); String responseBody = EntityUtils.toString(response.getEntity()); - assertThat(responseBody, containsString("knowledge_content")); + assertThat(responseBody, containsString("content")); request = new Request("GET", ".integration_knowledge-7/_mapping"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); response = client().performRequest(request); responseBody = EntityUtils.toString(response.getEntity()); - assertThat(responseBody, containsString("knowledge_content")); + assertThat(responseBody, containsString("content")); } } diff --git a/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java b/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java index fd710373a95c4..81497d5f784f0 100644 --- a/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java +++ b/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java @@ -28,6 +28,8 @@ public FleetRestIT(final ClientYamlTestCandidate testCandidate) { public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("x-pack-fleet") .module("x-pack-ilm") + .module("mapper-version") + .module("x-pack-inference") .module("data-streams") .setting("xpack.license.self_generated.type", "basic") .setting("xpack.security.enabled", "true") From 1a3d494452ba8ccc83c796a9eb5ba777198d5dc3 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Fri, 15 Aug 2025 13:20:48 -0600 Subject: [PATCH 23/25] changed index to externally mananged, and removed permissive warnings --- .../org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java | 3 --- .../src/main/java/org/elasticsearch/xpack/fleet/Fleet.java | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index 21c7646ffb7e1..a01ecfc4c8ee8 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; -import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentHelper; @@ -302,13 +301,11 @@ public void testCreationOfIntegrationKnowledge() throws Exception { assertEquals(200, response.getStatusLine().getStatusCode()); request = new Request("GET", ".integration_knowledge/_mapping"); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); response = client().performRequest(request); String responseBody = EntityUtils.toString(response.getEntity()); assertThat(responseBody, containsString("content")); request = new Request("GET", ".integration_knowledge-7/_mapping"); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); response = client().performRequest(request); responseBody = EntityUtils.toString(response.getEntity()); assertThat(responseBody, containsString("content")); diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 7adb64933e718..018082cb27832 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -277,7 +277,8 @@ private static SystemIndexDescriptor fleetIntegrationKnowledgeSystemIndexDescrip ); return SystemIndexDescriptor.builder() - .setType(Type.INTERNAL_MANAGED) + .setType(Type.EXTERNAL_MANAGED) + .setAllowedElasticProductOrigins(ALLOWED_PRODUCTS) .setOrigin(FLEET_ORIGIN) .setMappings(request.mappings()) .setSettings(request.settings()) From 72c11d375318b399e43cce5b7eb2f07e0727a496 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 15 Aug 2025 19:27:08 +0000 Subject: [PATCH 24/25] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index a01ecfc4c8ee8..6d1a2f2a9eb3a 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -9,7 +9,6 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; From 8a4f57277bc0e608a1120f4a7bef1656920567f2 Mon Sep 17 00:00:00 2001 From: Supplementing Date: Mon, 18 Aug 2025 14:08:07 -0600 Subject: [PATCH 25/25] made changes to support semantic search with the .md content being added in content field --- .../main/resources/fleet-integration-knowledge.json | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json index 24b5f5e815ee8..648fc4385c7d4 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json @@ -9,15 +9,19 @@ "_meta": { "version": "${fleet.version}", "managed_index_mappings_version": ${fleet.managed.index.version}, - "description": "Integration package knowledge base content storage", - "managed": true + "description": "Integration package knowledge base content storage" }, "properties": { "filename": { "type": "keyword" }, "content": { - "type": "semantic_text" + "type": "semantic_text", + "inference_id": ".elser-2-elasticsearch", + "model_settings": { + "service": "elasticsearch", + "task_type": "sparse_embedding" + } }, "version": { "type": "version"