Skip to content

Commit

Permalink
[Fleet] Task to publish Agent metrics (elastic#168435)
Browse files Browse the repository at this point in the history
## Summary

Closes elastic/ingest-dev#2396

Added a new kibana task that publishes Agent metrics every minute to
data streams installed by fleet_server package.

Opened the pr for review, there are a few things to finalize, but the
core logic won't change much.

To test locally:
- Install fleet_server package 1.4.0 from
[this](elastic/integrations#8145) pr to get the
mappings
- Start kibana locally, wait for a few minutes for the metrics task to
run (every minute)
- Go to discover, `metrics-*` index pattern, filter on
`data_stream.dataset: fleet_server.*`
- Expect data to be populated in `fleet_server.agent_status` and
`fleet_server.agent_versions` datasets.

<img width="1787" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/615af9df-fe4b-4c17-8c8c-88646c403a18">



### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
2 people authored and benakansara committed Oct 22, 2023
1 parent 3b8a116 commit cf0ec92
Show file tree
Hide file tree
Showing 16 changed files with 737 additions and 15 deletions.
3 changes: 3 additions & 0 deletions config/serverless.oblt.yml
Expand Up @@ -50,6 +50,9 @@ xpack.fleet.internal.registry.excludePackages: [
xpack.fleet.packages:
- name: apm
version: latest
# fleet_server package installed to publish agent metrics
- name: fleet_server
version: latest
## Disable APM UI components and API calls
xpack.apm.featureFlags.agentConfigurationAvailable: false
xpack.apm.featureFlags.configurableIndicesAvailable: false
Expand Down
4 changes: 4 additions & 0 deletions config/serverless.security.yml
Expand Up @@ -50,6 +50,10 @@ xpack.fleet.internal.registry.excludePackages: [
'symantec',
'cyberark',
]
# fleet_server package installed to publish agent metrics
xpack.fleet.packages:
- name: fleet_server
version: latest

xpack.ml.ad.enabled: true
xpack.ml.dfa.enabled: true
Expand Down
3 changes: 3 additions & 0 deletions test/functional/config.base.js
Expand Up @@ -34,6 +34,9 @@ export default async function ({ readConfigFile }) {

// to be re-enabled once kibana/issues/102552 is completed
'--xpack.reporting.enabled=false',

// disable fleet task that writes to metrics.fleet_server.* data streams, impacting functional tests
`--xpack.task_manager.unsafe.exclude_task_types=${JSON.stringify(['Fleet-Metrics-Task'])}`,
],
},

Expand Down
12 changes: 6 additions & 6 deletions x-pack/plugins/fleet/server/collectors/agent_collectors.ts
Expand Up @@ -58,13 +58,13 @@ export const getAgentUsage = async (
};
};

export interface AgentPerVersion {
version: string;
count: number;
}

export interface AgentData {
agents_per_version: Array<
{
version: string;
count: number;
} & AgentStatus
>;
agents_per_version: Array<AgentPerVersion & AgentStatus>;
agent_checkin_status: {
error: number;
degraded: number;
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/fleet/server/mocks/index.ts
Expand Up @@ -84,6 +84,7 @@ export const createAppContextStartContractMock = (
config$,
kibanaVersion: '8.99.0', // Fake version :)
kibanaBranch: 'main',
kibanaInstanceId: '1',
telemetryEventsSender: createMockTelemetryEventsSender(),
bulkActionsResolver: {} as any,
messageSigningService: createMessageSigningServiceMock(),
Expand Down
12 changes: 12 additions & 0 deletions x-pack/plugins/fleet/server/plugin.ts
Expand Up @@ -130,6 +130,8 @@ import { FleetActionsClient, type FleetActionsClientInterface } from './services
import type { FilesClientFactory } from './services/files/types';
import { PolicyWatcher } from './services/agent_policy_watch';
import { getPackageSpecTagId } from './services/epm/kibana/assets/tag_assets';
import { FleetMetricsTask } from './services/metrics/fleet_metrics_task';
import { fetchAgentMetrics } from './services/metrics/fetch_agent_metrics';

export interface FleetSetupDeps {
security: SecurityPluginSetup;
Expand Down Expand Up @@ -167,6 +169,7 @@ export interface FleetAppContext {
isProductionMode: PluginInitializerContext['env']['mode']['prod'];
kibanaVersion: PluginInitializerContext['env']['packageInfo']['version'];
kibanaBranch: PluginInitializerContext['env']['packageInfo']['branch'];
kibanaInstanceId: PluginInitializerContext['env']['instanceUuid'];
cloud?: CloudSetup;
logger?: Logger;
httpSetup?: HttpServiceSetup;
Expand Down Expand Up @@ -251,6 +254,7 @@ export class FleetPlugin
private isProductionMode: FleetAppContext['isProductionMode'];
private kibanaVersion: FleetAppContext['kibanaVersion'];
private kibanaBranch: FleetAppContext['kibanaBranch'];
private kibanaInstanceId: FleetAppContext['kibanaInstanceId'];
private httpSetup?: HttpServiceSetup;
private securitySetup!: SecurityPluginSetup;
private encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup;
Expand All @@ -259,6 +263,7 @@ export class FleetPlugin
private bulkActionsResolver?: BulkActionsResolver;
private fleetUsageSender?: FleetUsageSender;
private checkDeletedFilesTask?: CheckDeletedFilesTask;
private fleetMetricsTask?: FleetMetricsTask;

private agentService?: AgentService;
private packageService?: PackageService;
Expand All @@ -270,6 +275,7 @@ export class FleetPlugin
this.isProductionMode = this.initializerContext.env.mode.prod;
this.kibanaVersion = this.initializerContext.env.packageInfo.version;
this.kibanaBranch = this.initializerContext.env.packageInfo.branch;
this.kibanaInstanceId = this.initializerContext.env.instanceUuid;
this.logger = this.initializerContext.logger.get();
this.configInitialValue = this.initializerContext.config.get();
this.telemetryEventsSender = new TelemetryEventsSender(this.logger.get('telemetry_events'));
Expand Down Expand Up @@ -440,6 +446,10 @@ export class FleetPlugin
this.fleetUsageSender = new FleetUsageSender(deps.taskManager, core, fetch);
registerFleetUsageLogger(deps.taskManager, async () => fetchAgentsUsage(core, config));

const fetchAgents = async (abortController: AbortController) =>
await fetchAgentMetrics(core, abortController);
this.fleetMetricsTask = new FleetMetricsTask(deps.taskManager, fetchAgents);

const router: FleetRouter = core.http.createRouter<FleetRequestHandlerContext>();
// Allow read-only users access to endpoints necessary for Integrations UI
// Only some endpoints require superuser so we pass a raw IRouter here
Expand Down Expand Up @@ -490,6 +500,7 @@ export class FleetPlugin
isProductionMode: this.isProductionMode,
kibanaVersion: this.kibanaVersion,
kibanaBranch: this.kibanaBranch,
kibanaInstanceId: this.kibanaInstanceId,
httpSetup: this.httpSetup,
cloud: this.cloud,
logger: this.logger,
Expand All @@ -504,6 +515,7 @@ export class FleetPlugin
this.fleetUsageSender?.start(plugins.taskManager);
this.checkDeletedFilesTask?.start({ taskManager: plugins.taskManager });
startFleetUsageLogger(plugins.taskManager);
this.fleetMetricsTask?.start(plugins.taskManager, core.elasticsearch.client.asInternalUser);

const logger = appContextService.getLogger();

Expand Down
6 changes: 6 additions & 0 deletions x-pack/plugins/fleet/server/services/app_context.ts
Expand Up @@ -62,6 +62,7 @@ class AppContextService {
private isProductionMode: FleetAppContext['isProductionMode'] = false;
private kibanaVersion: FleetAppContext['kibanaVersion'] = kibanaPackageJson.version;
private kibanaBranch: FleetAppContext['kibanaBranch'] = kibanaPackageJson.branch;
private kibanaInstanceId: FleetAppContext['kibanaInstanceId'] = '';
private cloud?: CloudSetup;
private logger: Logger | undefined;
private httpSetup?: HttpServiceSetup;
Expand All @@ -86,6 +87,7 @@ class AppContextService {
this.logger = appContext.logger;
this.kibanaVersion = appContext.kibanaVersion;
this.kibanaBranch = appContext.kibanaBranch;
this.kibanaInstanceId = appContext.kibanaInstanceId;
this.httpSetup = appContext.httpSetup;
this.telemetryEventsSender = appContext.telemetryEventsSender;
this.savedObjectsTagging = appContext.savedObjectsTagging;
Expand Down Expand Up @@ -209,6 +211,10 @@ class AppContextService {
return this.kibanaBranch;
}

public getKibanaInstanceId() {
return this.kibanaInstanceId;
}

public addExternalCallback(type: ExternalCallback[0], callback: ExternalCallback[1]) {
if (!this.externalCallbacks.has(type)) {
this.externalCallbacks.set(type, new Set());
Expand Down
Expand Up @@ -22,12 +22,8 @@ export function getFilteredSearchPackages() {
}

export function getFilteredInstallPackages() {
const shouldFilterFleetServer = appContextService.getConfig()?.internal?.fleetServerStandalone;
const filtered: string[] = [];
// Do not allow to install Fleet server integration if configured to use standalone fleet server
if (shouldFilterFleetServer) {
filtered.push(FLEET_SERVER_PACKAGE);
}

const excludePackages = appContextService.getConfig()?.internal?.registry?.excludePackages ?? [];

return filtered.concat(excludePackages);
Expand Down
Expand Up @@ -332,7 +332,7 @@ describe('install', () => {
expect(response.status).toEqual('already_installed');
});

it('should not allow to install fleet_server if internal.fleetServerStandalone is configured', async () => {
it('should allow to install fleet_server if internal.fleetServerStandalone is configured', async () => {
jest.mocked(appContextService.getConfig).mockReturnValueOnce({
internal: {
fleetServerStandalone: true,
Expand All @@ -347,8 +347,7 @@ describe('install', () => {
esClient: {} as ElasticsearchClient,
});

expect(response.error).toBeDefined();
expect(response.error?.message).toMatch(/fleet_server installation is not authorized/);
expect(response.status).toEqual('installed');
});
});

Expand Down
@@ -0,0 +1,92 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
import { coreMock } from '@kbn/core/server/mocks';
import type { CoreSetup } from '@kbn/core/server';

import { fetchAgentMetrics } from './fetch_agent_metrics';

jest.mock('../../collectors/agent_collectors', () => {
return {
getAgentUsage: jest.fn().mockResolvedValue({}),
};
});

describe('fetchAgentMetrics', () => {
const { createSetup: coreSetupMock } = coreMock;
const abortController = new AbortController();
let mockCore: CoreSetup;
let esClient: ElasticsearchClientMock;

beforeEach(async () => {
mockCore = coreSetupMock();
const [{ elasticsearch }] = await mockCore.getStartServices();
esClient = elasticsearch.client.asInternalUser as ElasticsearchClientMock;
});

it('should fetch agent metrics', async () => {
esClient.search.mockResolvedValue({
took: 5,
timed_out: false,
_shards: {
total: 1,
successful: 1,
skipped: 0,
failed: 0,
},
hits: {
total: {
value: 0,
relation: 'eq',
},
hits: [],
},
aggregations: {
versions: {
buckets: [
{
key: '8.12.0',
doc_count: 1,
},
],
},
upgrade_details: {
buckets: [
{
key: 'UPG_REQUESTED',
doc_count: 1,
},
],
},
},
});

const result = await fetchAgentMetrics(mockCore, abortController);

expect(result).toEqual({
agents: {},
agents_per_version: [
{
version: '8.12.0',
count: 1,
},
],
upgrading_step: {
downloading: 0,
extracting: 0,
failed: 0,
replacing: 0,
requested: 1,
restarting: 0,
rollback: 0,
scheduled: 0,
watching: 0,
},
});
});
});

0 comments on commit cf0ec92

Please sign in to comment.