Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Ingest] Allow agent to send metadata compliant with ECS #64452

Merged
merged 3 commits into from
Apr 30, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions x-pack/plugins/ingest_manager/common/types/models/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ export interface AgentEvent {

export interface AgentEventSOAttributes extends AgentEvent, SavedObjectAttributes {}

type MetadataValue = string | AgentMetadata;

export interface AgentMetadata {
[x: string]: MetadataValue;
}
interface AgentBase {
type: AgentType;
active: boolean;
Expand All @@ -72,19 +77,17 @@ interface AgentBase {
config_revision?: number | null;
config_newest_revision?: number;
last_checkin?: string;
user_provided_metadata: AgentMetadata;
local_metadata: AgentMetadata;
}

export interface Agent extends AgentBase {
id: string;
current_error_events: AgentEvent[];
user_provided_metadata: Record<string, string>;
local_metadata: Record<string, string>;
access_api_key?: string;
status?: string;
}

export interface AgentSOAttributes extends AgentBase, SavedObjectAttributes {
user_provided_metadata: string;
local_metadata: string;
current_error_events?: string;
}
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ function useSuggestions(fieldPrefix: string, search: string) {
const res = (await data.indexPatterns.getFieldsForWildcard({
pattern: INDEX_NAME,
})) as IFieldType[];

if (!data || !data.autocomplete) {
throw new Error('Missing data plugin');
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { AgentMetadata } from '../../../../types';

export function flattenMetadata(metadata: AgentMetadata) {
return Object.entries(metadata).reduce((acc, [key, value]) => {
if (typeof value === 'string') {
acc[key] = value;

return acc;
}

Object.entries(flattenMetadata(value)).forEach(([flattenedKey, flattenedValue]) => {
acc[`${key}.${flattenedKey}`] = flattenedValue;
});

return acc;
}, {} as { [k: string]: string });
}
export function unflattenMetadata(flattened: { [k: string]: string }) {
const metadata: AgentMetadata = {};

Object.entries(flattened).forEach(([flattenedKey, flattenedValue]) => {
const keyParts = flattenedKey.split('.');
const lastKey = keyParts.pop();

if (!lastKey) {
throw new Error('Invalid metadata');
}

let metadataPart = metadata;
keyParts.forEach(keyPart => {
if (!metadataPart[keyPart]) {
metadataPart[keyPart] = {};
}

metadataPart = metadataPart[keyPart] as AgentMetadata;
});
metadataPart[lastKey] = flattenedValue;
});

return metadata;
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@ import {
} from '@elastic/eui';
import { MetadataForm } from './metadata_form';
import { Agent } from '../../../../types';
import { flattenMetadata } from './helper';

interface Props {
agent: Agent;
flyout: { hide: () => void };
}

export const AgentMetadataFlyout: React.FunctionComponent<Props> = ({ agent, flyout }) => {
const mapMetadata = (obj: { [key: string]: string } | undefined) => {
return Object.keys(obj || {}).map(key => ({
Expand All @@ -30,8 +32,8 @@ export const AgentMetadataFlyout: React.FunctionComponent<Props> = ({ agent, fly
}));
};

const localItems = mapMetadata(agent.local_metadata);
const userProvidedItems = mapMetadata(agent.user_provided_metadata);
const localItems = mapMetadata(flattenMetadata(agent.local_metadata));
const userProvidedItems = mapMetadata(flattenMetadata(agent.user_provided_metadata));

return (
<EuiFlyout onClose={() => flyout.hide()} size="s" aria-labelledby="flyoutTitle">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { useAgentRefresh } from '../hooks';
import { useInput, sendRequest } from '../../../../hooks';
import { Agent } from '../../../../types';
import { agentRouteService } from '../../../../services';
import { flattenMetadata, unflattenMetadata } from './helper';

function useAddMetadataForm(agent: Agent, done: () => void) {
const refreshAgent = useAgentRefresh();
Expand Down Expand Up @@ -66,15 +67,17 @@ function useAddMetadataForm(agent: Agent, done: () => void) {
isLoading: true,
});

const metadata = unflattenMetadata({
...flattenMetadata(agent.user_provided_metadata),
[keyInput.value]: valueInput.value,
});

try {
const { error } = await sendRequest({
path: agentRouteService.getUpdatePath(agent.id),
method: 'put',
body: JSON.stringify({
user_provided_metadata: {
...agent.user_provided_metadata,
[keyInput.value]: valueInput.value,
},
user_provided_metadata: metadata,
}),
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {

const columns = [
{
field: 'local_metadata.host',
field: 'local_metadata.host.hostname',
name: i18n.translate('xpack.ingestManager.agentList.hostColumnTitle', {
defaultMessage: 'Host',
}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export {
entries,
// Object types
Agent,
AgentMetadata,
AgentConfig,
NewAgentConfig,
AgentEvent,
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugins/ingest_manager/server/saved_objects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
enrolled_at: { type: 'date' },
access_api_key_id: { type: 'keyword' },
version: { type: 'keyword' },
user_provided_metadata: { type: 'text' },
local_metadata: { type: 'text' },
user_provided_metadata: { type: 'flattened' },
local_metadata: { type: 'flattened' },
config_id: { type: 'keyword' },
last_updated: { type: 'date' },
last_checkin: { type: 'date' },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
AgentAction,
AgentSOAttributes,
AgentEventSOAttributes,
AgentMetadata,
} from '../../types';

import { agentConfigService } from '../agent_config';
Expand All @@ -28,7 +29,7 @@ export async function agentCheckin(
const updateData: {
last_checkin: string;
default_api_key?: string;
local_metadata?: string;
local_metadata?: AgentMetadata;
current_error_events?: string;
} = {
last_checkin: new Date().toISOString(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ export async function updateAgent(
}
) {
await soClient.update<AgentSOAttributes>(AGENT_SAVED_OBJECT_TYPE, agentId, {
user_provided_metadata: JSON.stringify(data.userProvidedMetatada),
user_provided_metadata: data.userProvidedMetatada,
});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ export async function enroll(
config_id: configId,
type,
enrolled_at: enrolledAt,
user_provided_metadata: JSON.stringify(metadata?.userProvided ?? {}),
local_metadata: JSON.stringify(metadata?.local ?? {}),
user_provided_metadata: metadata?.userProvided ?? {},
local_metadata: metadata?.local ?? {},
current_error_events: undefined,
access_api_key_id: undefined,
last_checkin: undefined,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ export function savedObjectToAgent(so: SavedObject<AgentSOAttributes>): Agent {
current_error_events: so.attributes.current_error_events
? JSON.parse(so.attributes.current_error_events)
: [],
local_metadata: JSON.parse(so.attributes.local_metadata),
user_provided_metadata: JSON.parse(so.attributes.user_provided_metadata),
local_metadata: so.attributes.local_metadata,
user_provided_metadata: so.attributes.user_provided_metadata,
access_api_key: undefined,
status: undefined,
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ describe('Agent status service', () => {
type: AGENT_TYPE_PERMANENT,
attributes: {
active: false,
local_metadata: '{}',
user_provided_metadata: '{}',
local_metadata: {},
user_provided_metadata: {},
},
} as SavedObject<AgentSOAttributes>);
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
Expand All @@ -33,8 +33,8 @@ describe('Agent status service', () => {
type: AGENT_TYPE_PERMANENT,
attributes: {
active: true,
local_metadata: '{}',
user_provided_metadata: '{}',
local_metadata: {},
user_provided_metadata: {},
},
} as SavedObject<AgentSOAttributes>);
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/ingest_manager/server/types/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { ScopedClusterClient } from 'src/core/server';
export {
// Object types
Agent,
AgentMetadata,
AgentSOAttributes,
AgentStatus,
AgentType,
Expand Down
16 changes: 8 additions & 8 deletions x-pack/test/functional/es_archives/fleet/agents/data.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
"shared_id": "agent1_filebeat",
"config_id": "1",
"type": "PERMANENT",
"local_metadata": "{}",
"user_provided_metadata": "{}"
"local_metadata": {},
"user_provided_metadata": {}
}
}
}
Expand All @@ -30,8 +30,8 @@
"active": true,
"shared_id": "agent2_filebeat",
"type": "PERMANENT",
"local_metadata": "{}",
"user_provided_metadata": "{}"
"local_metadata": {},
"user_provided_metadata": {}
}
}
}
Expand All @@ -49,8 +49,8 @@
"active": true,
"shared_id": "agent3_metricbeat",
"type": "PERMANENT",
"local_metadata": "{}",
"user_provided_metadata": "{}"
"local_metadata": {},
"user_provided_metadata": {}
}
}
}
Expand All @@ -68,8 +68,8 @@
"active": true,
"shared_id": "agent4_metricbeat",
"type": "PERMANENT",
"local_metadata": "{}",
"user_provided_metadata": "{}"
"local_metadata": {},
"user_provided_metadata": {}
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions x-pack/test/functional/es_archives/fleet/agents/mappings.json
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@
"type": "date"
},
"local_metadata": {
"type": "text"
"type": "flattened"
},
"shared_id": {
"type": "keyword"
Expand All @@ -239,7 +239,7 @@
"type": "date"
},
"user_provided_metadata": {
"type": "text"
"type": "flattened"
},
"version": {
"type": "keyword"
Expand Down