From c26fd55de1a85f9f6efd87a03b779aa317c6e25d Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 30 Apr 2020 10:34:44 -0400 Subject: [PATCH] [Ingest] Allow aggent to send metadata compliant with ECS (#64452) --- .../common/types/models/agent.ts | 11 +++-- .../ingest_manager/components/search_bar.tsx | 1 - .../agent_details_page/components/helper.ts | 47 +++++++++++++++++++ .../components/metadata_flyout.tsx | 6 ++- .../components/metadata_form.tsx | 11 +++-- .../sections/fleet/agent_list_page/index.tsx | 2 +- .../ingest_manager/types/index.ts | 1 + .../ingest_manager/server/saved_objects.ts | 4 +- .../server/services/agents/checkin.ts | 3 +- .../server/services/agents/crud.ts | 2 +- .../server/services/agents/enroll.ts | 4 +- .../server/services/agents/saved_objects.ts | 4 +- .../server/services/agents/status.test.ts | 8 ++-- .../ingest_manager/server/types/index.tsx | 1 + .../es_archives/fleet/agents/data.json | 16 +++---- .../es_archives/fleet/agents/mappings.json | 4 +- 16 files changed, 91 insertions(+), 34 deletions(-) create mode 100644 x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/helper.ts diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent.ts b/x-pack/plugins/ingest_manager/common/types/models/agent.ts index fcd3955f3a32fc..e3ca7635fdb407 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent.ts @@ -60,6 +60,11 @@ export interface AgentEvent { export interface AgentEventSOAttributes extends AgentEvent, SavedObjectAttributes {} +type MetadataValue = string | AgentMetadata; + +export interface AgentMetadata { + [x: string]: MetadataValue; +} interface AgentBase { type: AgentType; active: boolean; @@ -72,19 +77,17 @@ interface AgentBase { config_revision?: number | null; config_newest_revision?: number; last_checkin?: string; + user_provided_metadata: AgentMetadata; + local_metadata: AgentMetadata; } export interface Agent extends AgentBase { id: string; current_error_events: AgentEvent[]; - user_provided_metadata: Record; - local_metadata: Record; access_api_key?: string; status?: string; } export interface AgentSOAttributes extends AgentBase, SavedObjectAttributes { - user_provided_metadata: string; - local_metadata: string; current_error_events?: string; } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx index 4429b9d8e6b829..1c9bd9107515d3 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx @@ -92,7 +92,6 @@ function useSuggestions(fieldPrefix: string, search: string) { const res = (await data.indexPatterns.getFieldsForWildcard({ pattern: INDEX_NAME, })) as IFieldType[]; - if (!data || !data.autocomplete) { throw new Error('Missing data plugin'); } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/helper.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/helper.ts new file mode 100644 index 00000000000000..508190fef0fc2b --- /dev/null +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/helper.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { AgentMetadata } from '../../../../types'; + +export function flattenMetadata(metadata: AgentMetadata) { + return Object.entries(metadata).reduce((acc, [key, value]) => { + if (typeof value === 'string') { + acc[key] = value; + + return acc; + } + + Object.entries(flattenMetadata(value)).forEach(([flattenedKey, flattenedValue]) => { + acc[`${key}.${flattenedKey}`] = flattenedValue; + }); + + return acc; + }, {} as { [k: string]: string }); +} +export function unflattenMetadata(flattened: { [k: string]: string }) { + const metadata: AgentMetadata = {}; + + Object.entries(flattened).forEach(([flattenedKey, flattenedValue]) => { + const keyParts = flattenedKey.split('.'); + const lastKey = keyParts.pop(); + + if (!lastKey) { + throw new Error('Invalid metadata'); + } + + let metadataPart = metadata; + keyParts.forEach(keyPart => { + if (!metadataPart[keyPart]) { + metadataPart[keyPart] = {}; + } + + metadataPart = metadataPart[keyPart] as AgentMetadata; + }); + metadataPart[lastKey] = flattenedValue; + }); + + return metadata; +} diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx index ee43385e601c28..aa6da36f8fb6c7 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx @@ -17,11 +17,13 @@ import { } from '@elastic/eui'; import { MetadataForm } from './metadata_form'; import { Agent } from '../../../../types'; +import { flattenMetadata } from './helper'; interface Props { agent: Agent; flyout: { hide: () => void }; } + export const AgentMetadataFlyout: React.FunctionComponent = ({ agent, flyout }) => { const mapMetadata = (obj: { [key: string]: string } | undefined) => { return Object.keys(obj || {}).map(key => ({ @@ -30,8 +32,8 @@ export const AgentMetadataFlyout: React.FunctionComponent = ({ agent, fly })); }; - const localItems = mapMetadata(agent.local_metadata); - const userProvidedItems = mapMetadata(agent.user_provided_metadata); + const localItems = mapMetadata(flattenMetadata(agent.local_metadata)); + const userProvidedItems = mapMetadata(flattenMetadata(agent.user_provided_metadata)); return ( flyout.hide()} size="s" aria-labelledby="flyoutTitle"> diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx index ce28bbdc590b06..af7e8c674db4c0 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx @@ -22,6 +22,7 @@ import { useAgentRefresh } from '../hooks'; import { useInput, sendRequest } from '../../../../hooks'; import { Agent } from '../../../../types'; import { agentRouteService } from '../../../../services'; +import { flattenMetadata, unflattenMetadata } from './helper'; function useAddMetadataForm(agent: Agent, done: () => void) { const refreshAgent = useAgentRefresh(); @@ -66,15 +67,17 @@ function useAddMetadataForm(agent: Agent, done: () => void) { isLoading: true, }); + const metadata = unflattenMetadata({ + ...flattenMetadata(agent.user_provided_metadata), + [keyInput.value]: valueInput.value, + }); + try { const { error } = await sendRequest({ path: agentRouteService.getUpdatePath(agent.id), method: 'put', body: JSON.stringify({ - user_provided_metadata: { - ...agent.user_provided_metadata, - [keyInput.value]: valueInput.value, - }, + user_provided_metadata: metadata, }), }); diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx index 23fe18b82468cb..05264c157434e0 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx @@ -238,7 +238,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => { const columns = [ { - field: 'local_metadata.host', + field: 'local_metadata.host.hostname', name: i18n.translate('xpack.ingestManager.agentList.hostColumnTitle', { defaultMessage: 'Host', }), diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts index 1508f4dfaa628b..d6483479a3f2fe 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts @@ -8,6 +8,7 @@ export { entries, // Object types Agent, + AgentMetadata, AgentConfig, NewAgentConfig, AgentEvent, diff --git a/x-pack/plugins/ingest_manager/server/saved_objects.ts b/x-pack/plugins/ingest_manager/server/saved_objects.ts index 90fe68e61bb1b4..89d8b9e173ffe0 100644 --- a/x-pack/plugins/ingest_manager/server/saved_objects.ts +++ b/x-pack/plugins/ingest_manager/server/saved_objects.ts @@ -56,8 +56,8 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = { enrolled_at: { type: 'date' }, access_api_key_id: { type: 'keyword' }, version: { type: 'keyword' }, - user_provided_metadata: { type: 'text' }, - local_metadata: { type: 'text' }, + user_provided_metadata: { type: 'flattened' }, + local_metadata: { type: 'flattened' }, config_id: { type: 'keyword' }, last_updated: { type: 'date' }, last_checkin: { type: 'date' }, diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts index c96a81ed9b7587..9b1565e7d74aab 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts @@ -11,6 +11,7 @@ import { AgentAction, AgentSOAttributes, AgentEventSOAttributes, + AgentMetadata, } from '../../types'; import { agentConfigService } from '../agent_config'; @@ -28,7 +29,7 @@ export async function agentCheckin( const updateData: { last_checkin: string; default_api_key?: string; - local_metadata?: string; + local_metadata?: AgentMetadata; current_error_events?: string; } = { last_checkin: new Date().toISOString(), diff --git a/x-pack/plugins/ingest_manager/server/services/agents/crud.ts b/x-pack/plugins/ingest_manager/server/services/agents/crud.ts index 175b92b75aca05..43fd5a3ce0ac94 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/crud.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/crud.ts @@ -103,7 +103,7 @@ export async function updateAgent( } ) { await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, { - user_provided_metadata: JSON.stringify(data.userProvidedMetatada), + user_provided_metadata: data.userProvidedMetatada, }); } diff --git a/x-pack/plugins/ingest_manager/server/services/agents/enroll.ts b/x-pack/plugins/ingest_manager/server/services/agents/enroll.ts index a34d2e03e9b3de..81afa70ecb818f 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/enroll.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/enroll.ts @@ -32,8 +32,8 @@ export async function enroll( config_id: configId, type, enrolled_at: enrolledAt, - user_provided_metadata: JSON.stringify(metadata?.userProvided ?? {}), - local_metadata: JSON.stringify(metadata?.local ?? {}), + user_provided_metadata: metadata?.userProvided ?? {}, + local_metadata: metadata?.local ?? {}, current_error_events: undefined, access_api_key_id: undefined, last_checkin: undefined, diff --git a/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts b/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts index b182662e0fb4e2..11beba1cd7e43e 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts @@ -19,8 +19,8 @@ export function savedObjectToAgent(so: SavedObject): Agent { current_error_events: so.attributes.current_error_events ? JSON.parse(so.attributes.current_error_events) : [], - local_metadata: JSON.parse(so.attributes.local_metadata), - user_provided_metadata: JSON.parse(so.attributes.user_provided_metadata), + local_metadata: so.attributes.local_metadata, + user_provided_metadata: so.attributes.user_provided_metadata, access_api_key: undefined, status: undefined, }; diff --git a/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts b/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts index b6de083cbe0cb2..8140b1e6de470f 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts @@ -18,8 +18,8 @@ describe('Agent status service', () => { type: AGENT_TYPE_PERMANENT, attributes: { active: false, - local_metadata: '{}', - user_provided_metadata: '{}', + local_metadata: {}, + user_provided_metadata: {}, }, } as SavedObject); const status = await getAgentStatusById(mockSavedObjectsClient, 'id'); @@ -33,8 +33,8 @@ describe('Agent status service', () => { type: AGENT_TYPE_PERMANENT, attributes: { active: true, - local_metadata: '{}', - user_provided_metadata: '{}', + local_metadata: {}, + user_provided_metadata: {}, }, } as SavedObject); const status = await getAgentStatusById(mockSavedObjectsClient, 'id'); diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index a7019ebc0a271f..27ed1de8499871 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -8,6 +8,7 @@ import { ScopedClusterClient } from 'src/core/server'; export { // Object types Agent, + AgentMetadata, AgentSOAttributes, AgentStatus, AgentType, diff --git a/x-pack/test/functional/es_archives/fleet/agents/data.json b/x-pack/test/functional/es_archives/fleet/agents/data.json index 3fe4f828ba128b..d22e3cd3fecdda 100644 --- a/x-pack/test/functional/es_archives/fleet/agents/data.json +++ b/x-pack/test/functional/es_archives/fleet/agents/data.json @@ -11,8 +11,8 @@ "shared_id": "agent1_filebeat", "config_id": "1", "type": "PERMANENT", - "local_metadata": "{}", - "user_provided_metadata": "{}" + "local_metadata": {}, + "user_provided_metadata": {} } } } @@ -30,8 +30,8 @@ "active": true, "shared_id": "agent2_filebeat", "type": "PERMANENT", - "local_metadata": "{}", - "user_provided_metadata": "{}" + "local_metadata": {}, + "user_provided_metadata": {} } } } @@ -49,8 +49,8 @@ "active": true, "shared_id": "agent3_metricbeat", "type": "PERMANENT", - "local_metadata": "{}", - "user_provided_metadata": "{}" + "local_metadata": {}, + "user_provided_metadata": {} } } } @@ -68,8 +68,8 @@ "active": true, "shared_id": "agent4_metricbeat", "type": "PERMANENT", - "local_metadata": "{}", - "user_provided_metadata": "{}" + "local_metadata": {}, + "user_provided_metadata": {} } } } diff --git a/x-pack/test/functional/es_archives/fleet/agents/mappings.json b/x-pack/test/functional/es_archives/fleet/agents/mappings.json index 5d5d373797d4cf..409cc3c689eafc 100644 --- a/x-pack/test/functional/es_archives/fleet/agents/mappings.json +++ b/x-pack/test/functional/es_archives/fleet/agents/mappings.json @@ -227,7 +227,7 @@ "type": "date" }, "local_metadata": { - "type": "text" + "type": "flattened" }, "shared_id": { "type": "keyword" @@ -239,7 +239,7 @@ "type": "date" }, "user_provided_metadata": { - "type": "text" + "type": "flattened" }, "version": { "type": "keyword"