Skip to content

Commit

Permalink
[7.9] [Security Solution] [Detections] Log message enhancements (#78429…
Browse files Browse the repository at this point in the history 
…) (#78672)

* adds missing buildRuleMessage to debug logs to display rule id, name, etc. in logs

* add buildRuleMessage fn to params

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
# Conflicts:
#	x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts
#	x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts
#	x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts
  • Loading branch information
@dhurley14
dhurley14 committed Sep 28, 2020
1 parent 7e12fc7 commit 4eefb83
Show file tree
Hide file tree
Showing 9 changed files with 55 additions and 8 deletions.
6 changes: 4 additions & 2 deletions ...k/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
View file
Expand Up @@ -14,6 +14,7 @@ import { RuleAlertAction } from '../../../../common/detection_engine/types';
import { RuleTypeParams, RefreshTypes } from '../types';
import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create';
import { AnomalyResults, Anomaly } from '../../machine_learning';
import { BuildRuleMessage } from './rule_messages';

interface BulkCreateMlSignalsParams {
actions: RuleAlertAction[];
Expand All @@ -33,6 +34,7 @@ interface BulkCreateMlSignalsParams {
refresh: RefreshTypes;
tags: string[];
throttle: string;
buildRuleMessage: BuildRuleMessage;
}

interface EcsAnomaly extends Anomaly {
Expand Down Expand Up @@ -85,6 +87,6 @@ export const bulkCreateMlSignals = async (
): Promise<SingleBulkCreateResponse> => {
const anomalyResults = params.someResult;
const ecsResults = transformAnomalyResultsToEcs(anomalyResults);

return singleBulkCreate({ ...params, filteredEvents: ecsResults });
const buildRuleMessage = params.buildRuleMessage;
return singleBulkCreate({ ...params, filteredEvents: ecsResults, buildRuleMessage });
};
5 changes: 4 additions & 1 deletion ...ns/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts
View file
Expand Up @@ -15,6 +15,7 @@ import { RuleAlertAction } from '../../../../common/detection_engine/types';
import { RuleTypeParams, RefreshTypes } from '../types';
import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create';
import { SignalSearchResponse } from './types';
import { BuildRuleMessage } from './rule_messages';

// used to generate constant Threshold Signals ID when run with the same params
const NAMESPACE_ID = '0684ec03-7201-4ee0-8ee0-3a3f6b2479b2';
Expand All @@ -40,6 +41,7 @@ interface BulkCreateThresholdSignalsParams {
tags: string[];
throttle: string;
startedAt: Date;
buildRuleMessage: BuildRuleMessage;
}

interface FilterObject {
Expand Down Expand Up @@ -194,6 +196,7 @@ export const bulkCreateThresholdSignals = async (
params.ruleParams.threshold!,
params.ruleParams.ruleId
);
const buildRuleMessage = params.buildRuleMessage;

return singleBulkCreate({ ...params, filteredEvents: ecsResults });
return singleBulkCreate({ ...params, filteredEvents: ecsResults, buildRuleMessage });
};
4 changes: 4 additions & 0 deletions ...k/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts
View file
Expand Up @@ -12,6 +12,7 @@ import { singleSearchAfter } from './single_search_after';
import { AlertServices } from '../../../../../alerts/server';
import { Logger } from '../../../../../../../src/core/server';
import { SignalSearchResponse } from './types';
import { BuildRuleMessage } from './rule_messages';

interface FindThresholdSignalsParams {
from: string;
Expand All @@ -21,6 +22,7 @@ interface FindThresholdSignalsParams {
logger: Logger;
filter: unknown;
threshold: Threshold;
buildRuleMessage: BuildRuleMessage;
}

export const findThresholdSignals = async ({
Expand All @@ -31,6 +33,7 @@ export const findThresholdSignals = async ({
logger,
filter,
threshold,
buildRuleMessage,
}: FindThresholdSignalsParams): Promise<{
searchResult: SignalSearchResponse;
searchDuration: string;
Expand Down Expand Up @@ -58,5 +61,6 @@ export const findThresholdSignals = async ({
logger,
filter,
pageSize: 0,
buildRuleMessage,
});
};
2 changes: 2 additions & 0 deletions ...plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts
View file
Expand Up @@ -126,6 +126,7 @@ export const searchAfterAndBulkCreate = async ({
searchDuration,
}: { searchResult: SignalSearchResponse; searchDuration: string } = await singleSearchAfter(
{
buildRuleMessage,
searchAfterSortId: sortId,
index: inputIndexPattern,
from: tuple.from.toISOString(),
Expand Down Expand Up @@ -205,6 +206,7 @@ export const searchAfterAndBulkCreate = async ({
bulkCreateDuration: bulkDuration,
createdItemsCount: createdCount,
} = await singleBulkCreate({
buildRuleMessage,
filteredEvents,
ruleParams,
services,
Expand Down
3 changes: 3 additions & 0 deletions ...k/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
View file
Expand Up @@ -240,6 +240,7 @@ export const signalRulesAlertType = ({
enabled,
refresh,
tags,
buildRuleMessage,
});
result.success = success;
result.createdSignalsCount = createdItemsCount;
Expand Down Expand Up @@ -267,6 +268,7 @@ export const signalRulesAlertType = ({
logger,
filter: esFilter,
threshold,
buildRuleMessage,
});

const {
Expand Down Expand Up @@ -294,6 +296,7 @@ export const signalRulesAlertType = ({
enabled,
refresh,
tags,
buildRuleMessage,
});
result.success = success;
result.createdSignalsCount = createdItemsCount;
Expand Down
13 changes: 13 additions & 0 deletions .../plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts
View file
Expand Up @@ -19,7 +19,14 @@ import {
import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants';
import { singleBulkCreate, filterDuplicateRules } from './single_bulk_create';
import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
import { buildRuleMessageFactory } from './rule_messages';

const buildRuleMessage = buildRuleMessageFactory({
id: 'fake id',
ruleId: 'fake rule id',
index: 'fakeindex',
name: 'fake name',
});
describe('singleBulkCreate', () => {
const mockService: AlertServicesMock = alertsMock.createAlertServices();

Expand Down Expand Up @@ -158,6 +165,7 @@ describe('singleBulkCreate', () => {
refresh: false,
tags: ['some fake tag 1', 'some fake tag 2'],
throttle: 'no_actions',
buildRuleMessage,
});
expect(success).toEqual(true);
expect(createdItemsCount).toEqual(0);
Expand Down Expand Up @@ -192,6 +200,7 @@ describe('singleBulkCreate', () => {
refresh: false,
tags: ['some fake tag 1', 'some fake tag 2'],
throttle: 'no_actions',
buildRuleMessage,
});
expect(success).toEqual(true);
expect(createdItemsCount).toEqual(0);
Expand All @@ -218,6 +227,7 @@ describe('singleBulkCreate', () => {
refresh: false,
tags: ['some fake tag 1', 'some fake tag 2'],
throttle: 'no_actions',
buildRuleMessage,
});
expect(success).toEqual(true);
expect(createdItemsCount).toEqual(0);
Expand Down Expand Up @@ -245,6 +255,7 @@ describe('singleBulkCreate', () => {
refresh: false,
tags: ['some fake tag 1', 'some fake tag 2'],
throttle: 'no_actions',
buildRuleMessage,
});

expect(mockLogger.error).not.toHaveBeenCalled();
Expand Down Expand Up @@ -274,6 +285,7 @@ describe('singleBulkCreate', () => {
refresh: false,
tags: ['some fake tag 1', 'some fake tag 2'],
throttle: 'no_actions',
buildRuleMessage,
});

expect(mockLogger.error).toHaveBeenCalled();
Expand Down Expand Up @@ -369,6 +381,7 @@ describe('singleBulkCreate', () => {
refresh: false,
tags: ['some fake tag 1', 'some fake tag 2'],
throttle: 'no_actions',
buildRuleMessage,
});
expect(success).toEqual(true);
expect(createdItemsCount).toEqual(1);
Expand Down
15 changes: 11 additions & 4 deletions x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts
View file
Expand Up @@ -12,6 +12,7 @@ import { RuleAlertAction } from '../../../../common/detection_engine/types';
import { RuleTypeParams, RefreshTypes } from '../types';
import { generateId, makeFloatString, errorAggregator } from './utils';
import { buildBulkBody } from './build_bulk_body';
import { BuildRuleMessage } from './rule_messages';
import { Logger } from '../../../../../../../src/core/server';

interface SingleBulkCreateParams {
Expand All @@ -32,6 +33,7 @@ interface SingleBulkCreateParams {
tags: string[];
throttle: string;
refresh: RefreshTypes;
buildRuleMessage: BuildRuleMessage;
}

/**
Expand Down Expand Up @@ -64,6 +66,7 @@ export interface SingleBulkCreateResponse {

// Bulk Index documents.
export const singleBulkCreate = async ({
buildRuleMessage,
filteredEvents,
ruleParams,
services,
Expand All @@ -83,9 +86,9 @@ export const singleBulkCreate = async ({
throttle,
}: SingleBulkCreateParams): Promise<SingleBulkCreateResponse> => {
filteredEvents.hits.hits = filterDuplicateRules(id, filteredEvents);
logger.debug(`about to bulk create ${filteredEvents.hits.hits.length} events`);
logger.debug(buildRuleMessage(`about to bulk create ${filteredEvents.hits.hits.length} events`));
if (filteredEvents.hits.hits.length === 0) {
logger.debug(`all events were duplicates`);
logger.debug(buildRuleMessage(`all events were duplicates`));
return { success: true, createdItemsCount: 0 };
}
// index documents after creating an ID based on the
Expand Down Expand Up @@ -132,8 +135,12 @@ export const singleBulkCreate = async ({
body: bulkBody,
});
const end = performance.now();
logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`);
logger.debug(`took property says bulk took: ${response.took} milliseconds`);
logger.debug(
buildRuleMessage(
`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`
)
);
logger.debug(buildRuleMessage(`took property says bulk took: ${response.took} milliseconds`));

if (response.errors) {
const duplicateSignalsCount = countBy(response.items, 'create.status')['409'];
Expand Down
10 changes: 10 additions & 0 deletions ...plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts
View file
Expand Up @@ -11,7 +11,14 @@ import {
} from './__mocks__/es_results';
import { singleSearchAfter } from './single_search_after';
import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
import { buildRuleMessageFactory } from './rule_messages';

const buildRuleMessage = buildRuleMessageFactory({
id: 'fake id',
ruleId: 'fake rule id',
index: 'fakeindex',
name: 'fake name',
});
describe('singleSearchAfter', () => {
const mockService: AlertServicesMock = alertsMock.createAlertServices();

Expand All @@ -32,6 +39,7 @@ describe('singleSearchAfter', () => {
pageSize: 1,
filter: undefined,
timestampOverride: undefined,
buildRuleMessage,
});
expect(searchResult).toEqual(sampleDocSearchResultsNoSortId);
});
Expand All @@ -48,6 +56,7 @@ describe('singleSearchAfter', () => {
pageSize: 1,
filter: undefined,
timestampOverride: undefined,
buildRuleMessage,
});
expect(searchResult).toEqual(sampleDocSearchResultsWithSortId);
});
Expand All @@ -67,6 +76,7 @@ describe('singleSearchAfter', () => {
pageSize: 1,
filter: undefined,
timestampOverride: undefined,
buildRuleMessage,
})
).rejects.toThrow('Fake Error');
});
Expand Down
5 changes: 4 additions & 1 deletion x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts
View file
Expand Up @@ -8,6 +8,7 @@ import { performance } from 'perf_hooks';
import { AlertServices } from '../../../../../alerts/server';
import { Logger } from '../../../../../../../src/core/server';
import { SignalSearchResponse } from './types';
import { BuildRuleMessage } from './rule_messages';
import { buildEventsSearchQuery } from './build_events_query';
import { makeFloatString } from './utils';
import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas';
Expand All @@ -23,6 +24,7 @@ interface SingleSearchAfterParams {
pageSize: number;
filter: unknown;
timestampOverride: TimestampOverrideOrUndefined;
buildRuleMessage: BuildRuleMessage;
}

// utilize search_after for paging results into bulk.
Expand All @@ -37,6 +39,7 @@ export const singleSearchAfter = async ({
logger,
pageSize,
timestampOverride,
buildRuleMessage,
}: SingleSearchAfterParams): Promise<{
searchResult: SignalSearchResponse;
searchDuration: string;
Expand All @@ -61,7 +64,7 @@ export const singleSearchAfter = async ({
const end = performance.now();
return { searchResult: nextSearchAfterResult, searchDuration: makeFloatString(end - start) };
} catch (exc) {
logger.error(`[-] nextSearchAfter threw an error ${exc}`);
logger.error(buildRuleMessage(`[-] nextSearchAfter threw an error ${exc}`));
throw exc;
}
};

0 comments on commit 4eefb83

Please sign in to comment.