Skip to content

Commit

Permalink
[APM] Reenable and stabilize APM correlations API integration tests. (#…
Browse files Browse the repository at this point in the history 
…187444)

## Summary

Fixes #176544.
Fixes #187421.
Fixes #176119.
Fixes #176425.
Fixes #175855.
Fixes #175911.
Fixes #176780.

Follow up to #186182.

Reenables and stabilizes APM correlations API integration tests.

Review hint: View with the `w=1` flag to ignore whitespace changes:
https://github.com/elastic/kibana/pull/187444/files?w=1

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
  • Loading branch information
@walterra
walterra committed Jul 3, 2024
1 parent bd41c65 commit 39d3e1a
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 146 deletions.
225 changes: 109 additions & 116 deletions x-pack/test/apm_api_integration/tests/correlations/failed_transactions.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
* 2.0.
*/

import { orderBy } from 'lodash';
import expect from '@kbn/expect';

import type { FailedTransactionsCorrelationsResponse } from '@kbn/apm-plugin/common/correlations/failed_transactions_correlations/types';
import { EVENT_OUTCOME } from '@kbn/apm-plugin/common/es_fields/apm';
import { EventOutcome } from '@kbn/apm-plugin/common/event_outcome';
Expand All @@ -27,7 +27,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
kuery: '',
});

registry.when.skip('failed transactions without data', { config: 'trial', archives: [] }, () => {
registry.when('failed transactions without data', { config: 'trial', archives: [] }, () => {
it('handles the empty state', async () => {
const overallDistributionResponse = await apmApiClient.readUser({
endpoint: 'POST /internal/apm/latency/overall_distribution/transactions',
Expand Down Expand Up @@ -104,127 +104,120 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
});

// FLAKY: https://github.com/elastic/kibana/issues/176544
registry.when.skip(
'failed transactions with data',
{ config: 'trial', archives: ['8.0.0'] },
() => {
it('runs queries and returns results', async () => {
const overallDistributionResponse = await apmApiClient.readUser({
endpoint: 'POST /internal/apm/latency/overall_distribution/transactions',
params: {
body: {
...getOptions(),
percentileThreshold: 95,
chartType: LatencyDistributionChartType.failedTransactionsCorrelations,
},
registry.when('failed transactions with data', { config: 'trial', archives: ['8.0.0'] }, () => {
it('runs queries and returns results', async () => {
const overallDistributionResponse = await apmApiClient.readUser({
endpoint: 'POST /internal/apm/latency/overall_distribution/transactions',
params: {
body: {
...getOptions(),
percentileThreshold: 95,
chartType: LatencyDistributionChartType.failedTransactionsCorrelations,
},
});
},
});

expect(overallDistributionResponse.status).to.eql(
200,
`Expected status to be '200', got '${overallDistributionResponse.status}'`
);

expect(overallDistributionResponse.status).to.eql(
200,
`Expected status to be '200', got '${overallDistributionResponse.status}'`
);

const errorDistributionResponse = await apmApiClient.readUser({
endpoint: 'POST /internal/apm/latency/overall_distribution/transactions',
params: {
body: {
...getOptions(),
percentileThreshold: 95,
termFilters: [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }],
chartType: LatencyDistributionChartType.failedTransactionsCorrelations,
},
const errorDistributionResponse = await apmApiClient.readUser({
endpoint: 'POST /internal/apm/latency/overall_distribution/transactions',
params: {
body: {
...getOptions(),
percentileThreshold: 95,
termFilters: [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }],
chartType: LatencyDistributionChartType.failedTransactionsCorrelations,
},
});
},
});

expect(errorDistributionResponse.status).to.eql(
200,
`Expected status to be '200', got '${errorDistributionResponse.status}'`
);
expect(errorDistributionResponse.status).to.eql(
200,
`Expected status to be '200', got '${errorDistributionResponse.status}'`
);

const fieldCandidatesResponse = await apmApiClient.readUser({
endpoint: 'GET /internal/apm/correlations/field_candidates/transactions',
params: {
query: getOptions(),
},
});
const fieldCandidatesResponse = await apmApiClient.readUser({
endpoint: 'GET /internal/apm/correlations/field_candidates/transactions',
params: {
query: getOptions(),
},
});

expect(fieldCandidatesResponse.status).to.eql(
200,
`Expected status to be '200', got '${fieldCandidatesResponse.status}'`
);

const fieldCandidates = fieldCandidatesResponse.body?.fieldCandidates.filter(
(t) => !(t === EVENT_OUTCOME)
);

// Identified 68 fieldCandidates.
expect(fieldCandidates.length).to.eql(
68,
`Expected field candidates length to be '68', got '${fieldCandidates.length}'`
);

const failedTransactionsCorrelationsResponse = await apmApiClient.readUser({
endpoint: 'POST /internal/apm/correlations/p_values/transactions',
params: {
body: {
...getOptions(),
fieldCandidates,
},
expect(fieldCandidatesResponse.status).to.eql(
200,
`Expected status to be '200', got '${fieldCandidatesResponse.status}'`
);

const fieldCandidates = fieldCandidatesResponse.body?.fieldCandidates.filter(
(t) => !(t === EVENT_OUTCOME)
);

// Identified 80 fieldCandidates.
expect(fieldCandidates.length).to.eql(
80,
`Expected field candidates length to be '80', got '${fieldCandidates.length}'`
);

const failedTransactionsCorrelationsResponse = await apmApiClient.readUser({
endpoint: 'POST /internal/apm/correlations/p_values/transactions',
params: {
body: {
...getOptions(),
fieldCandidates,
},
},
});

expect(failedTransactionsCorrelationsResponse.status).to.eql(
200,
`Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'`
);

const fieldsToSample = new Set<string>();
if (failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations.length > 0) {
failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations.forEach((d) => {
fieldsToSample.add(d.fieldName);
});
}

expect(failedTransactionsCorrelationsResponse.status).to.eql(
200,
`Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'`
);

const fieldsToSample = new Set<string>();
if (
failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations.length > 0
) {
failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations.forEach(
(d) => {
fieldsToSample.add(d.fieldName);
}
);
}

const finalRawResponse: FailedTransactionsCorrelationsResponse = {
ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning,
percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue,
overallHistogram: overallDistributionResponse.body?.overallHistogram,
errorHistogram: errorDistributionResponse.body?.overallHistogram,
failedTransactionsCorrelations:
failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations,
};

expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875);
expect(finalRawResponse?.errorHistogram?.length).to.be(101);
expect(finalRawResponse?.overallHistogram?.length).to.be(101);

expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql(
30,
`Expected 30 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.`
);

const sortedCorrelations = finalRawResponse?.failedTransactionsCorrelations?.sort(
(a, b) => b.score - a.score
);
const correlation = sortedCorrelations?.[0];

expect(typeof correlation).to.be('object');
expect(correlation?.doc_count).to.be(31);
expect(correlation?.score).to.be(83.70467673605746);
expect(correlation?.bg_count).to.be(31);
expect(correlation?.fieldName).to.be('http.response.status_code');
expect(correlation?.fieldValue).to.be(500);
expect(typeof correlation?.pValue).to.be('number');
expect(typeof correlation?.normalizedScore).to.be('number');
expect(typeof correlation?.failurePercentage).to.be('number');
expect(typeof correlation?.successPercentage).to.be('number');
});
}
);
const finalRawResponse: FailedTransactionsCorrelationsResponse = {
ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning,
percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue,
overallHistogram: overallDistributionResponse.body?.overallHistogram,
errorHistogram: errorDistributionResponse.body?.overallHistogram,
failedTransactionsCorrelations:
failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations,
};

expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875);
expect(finalRawResponse?.errorHistogram?.length).to.be(101);
expect(finalRawResponse?.overallHistogram?.length).to.be(101);

expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql(
29,
`Expected 29 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.`
);

const sortedCorrelations = orderBy(
finalRawResponse?.failedTransactionsCorrelations,
['score', 'fieldName', 'fieldValue'],
['desc', 'asc', 'asc']
);
const correlation = sortedCorrelations?.[0];

expect(typeof correlation).to.be('object');
expect(correlation?.doc_count).to.be(31);
expect(correlation?.score).to.be(83.70467673605746);
expect(correlation?.bg_count).to.be(31);
expect(correlation?.fieldName).to.be('transaction.result');
expect(correlation?.fieldValue).to.be('HTTP 5xx');
expect(typeof correlation?.pValue).to.be('number');
expect(typeof correlation?.normalizedScore).to.be('number');
expect(typeof correlation?.failurePercentage).to.be('number');
expect(typeof correlation?.successPercentage).to.be('number');
});
});
}
12 changes: 6 additions & 6 deletions x-pack/test/apm_api_integration/tests/correlations/field_candidates.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,21 @@ export default function ApiTest({ getService }: FtrProviderContext) {
},
});

// FLAKY: https://github.com/elastic/kibana/issues/187421
registry.when.skip('field candidates without data', { config: 'trial', archives: [] }, () => {
registry.when('field candidates without data', { config: 'trial', archives: [] }, () => {
it('handles the empty state', async () => {
const response = await apmApiClient.readUser({
endpoint,
...getOptions(),
});

expect(response.status).to.be(200);
expect(response.body?.fieldCandidates.length).to.be(14);
// If the source indices are empty, there will be no field candidates
// because of the `include_empty_fields: false` option in the query.
expect(response.body?.fieldCandidates.length).to.be(0);
});
});

// FLAKY: https://github.com/elastic/kibana/issues/176119
registry.when.skip(
registry.when(
'field candidates with data and default args',
{ config: 'trial', archives: ['8.0.0'] },
() => {
Expand All @@ -50,7 +50,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});

expect(response.status).to.eql(200);
expect(response.body?.fieldCandidates.length).to.be(69);
expect(response.body?.fieldCandidates.length).to.be(81);
});
}
);
Expand Down
3 changes: 1 addition & 2 deletions x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
});

// FLAKY: https://github.com/elastic/kibana/issues/176425
registry.when.skip(
registry.when(
'field value pairs with data and default args',
{ config: 'trial', archives: ['8.0.0'] },
() => {
Expand Down
38 changes: 20 additions & 18 deletions x-pack/test/apm_api_integration/tests/correlations/latency.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { chunk } from 'lodash';
import { chunk, orderBy } from 'lodash';

import expect from '@kbn/expect';

Expand Down Expand Up @@ -107,8 +107,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
}
);

// FLAKY: https://github.com/elastic/kibana/issues/175855
registry.when.skip(
registry.when(
'correlations latency with data and opbeans-node args',
{ config: 'trial', archives: ['8.0.0'] },
() => {
Expand Down Expand Up @@ -142,10 +141,10 @@ export default function ApiTest({ getService }: FtrProviderContext) {
`Expected status to be '200', got '${fieldCandidatesResponse.status}'`
);

// Identified 69 fieldCandidates.
// Identified 81 fieldCandidates.
expect(fieldCandidatesResponse.body?.fieldCandidates.length).to.eql(
69,
`Expected field candidates length to be '69', got '${fieldCandidatesResponse.body?.fieldCandidates.length}'`
81,
`Expected field candidates length to be '81', got '${fieldCandidatesResponse.body?.fieldCandidates.length}'`
);

const fieldValuePairsResponse = await apmApiClient.readUser({
Expand All @@ -163,15 +162,15 @@ export default function ApiTest({ getService }: FtrProviderContext) {
`Expected status to be '200', got '${fieldValuePairsResponse.status}'`
);

// Identified 379 fieldValuePairs.
// Identified 374 fieldValuePairs.
expect(fieldValuePairsResponse.body?.fieldValuePairs.length).to.eql(
379,
`Expected field value pairs length to be '379', got '${fieldValuePairsResponse.body?.fieldValuePairs.length}'`
374,
`Expected field value pairs length to be '374', got '${fieldValuePairsResponse.body?.fieldValuePairs.length}'`
);

// This replicates the code used in the `useLatencyCorrelations` hook to chunk requests for correlation analysis.
// Tests turned out to be flaky and occasionally overload ES with a `search_phase_execution_exception`
// when all 379 field value pairs from above are queried in parallel.
// when all 374 field value pairs from above are queried in parallel.
// The chunking sends 10 field value pairs with each request to the Kibana API endpoint.
// Kibana itself will then run those 10 requests in parallel against ES.
const latencyCorrelations: LatencyCorrelation[] = [];
Expand Down Expand Up @@ -232,20 +231,23 @@ export default function ApiTest({ getService }: FtrProviderContext) {
expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875);
expect(finalRawResponse?.overallHistogram?.length).to.be(101);

// Identified 13 significant correlations out of 379 field/value pairs.
// Identified 13 significant correlations out of 374 field/value pairs.
expect(finalRawResponse?.latencyCorrelations?.length).to.eql(
13,
`Expected 13 identified correlations, got ${finalRawResponse?.latencyCorrelations?.length}.`
);

const correlation = finalRawResponse?.latencyCorrelations?.sort(
(a, b) => b.correlation - a.correlation
)[0];
const sortedCorrelations = orderBy(
finalRawResponse?.latencyCorrelations,
['score', 'fieldName', 'fieldValue'],
['desc', 'asc', 'asc']
);
const correlation = sortedCorrelations?.[0];
expect(typeof correlation).to.be('object');
expect(correlation?.fieldName).to.be('transaction.result');
expect(correlation?.fieldValue).to.be('success');
expect(correlation?.correlation).to.be(0.6275246559191225);
expect(correlation?.ksTest).to.be(4.806503252860024e-13);
expect(correlation?.fieldName).to.be('agent.hostname');
expect(correlation?.fieldValue).to.be('rum-js');
expect(correlation?.correlation).to.be(0.34798078715348596);
expect(correlation?.ksTest).to.be(1.9848961005439386e-12);
expect(correlation?.histogram?.length).to.be(101);
});
}
Expand Down
3 changes: 1 addition & 2 deletions x-pack/test/apm_api_integration/tests/correlations/p_values.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
});

// FLAKY: https://github.com/elastic/kibana/issues/175911
registry.when.skip(
registry.when(
'p values with data and default args',
{ config: 'trial', archives: ['8.0.0'] },
() => {
Expand Down
Loading

0 comments on commit 39d3e1a

Please sign in to comment.