diff --git a/.buildkite/pipelines/flaky_tests/pipeline.ts b/.buildkite/pipelines/flaky_tests/pipeline.ts index 89505065b5809a..1d12b2c8407448 100644 --- a/.buildkite/pipelines/flaky_tests/pipeline.ts +++ b/.buildkite/pipelines/flaky_tests/pipeline.ts @@ -162,10 +162,11 @@ for (const testSuite of testSuites) { `Group configuration was not found in groups.json for the following cypress suite: {${suiteName}}.` ); } + const agentQueue = suiteName.includes('defend_workflows') ? 'n2-4-virt' : 'n2-4-spot'; steps.push({ command: `.buildkite/scripts/steps/functional/${suiteName}.sh`, label: group.name, - agents: { queue: 'n2-4-spot' }, + agents: { queue: agentQueue }, depends_on: 'build', parallelism: testSuite.count, concurrency, diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts index f7257060e4ca97..b94f389958f9f8 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts @@ -20,8 +20,7 @@ import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; -// FLAKY: https://github.com/elastic/kibana/issues/168340 -describe.skip( +describe( 'Automated Response Actions', { tags: [ @@ -76,8 +75,7 @@ describe.skip( disableExpandableFlyoutAdvancedSettings(); }); - // FLAKY: https://github.com/elastic/kibana/issues/168427 - describe.skip('From alerts', () => { + describe('From alerts', () => { let ruleId: string; let ruleName: string; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts index b7d9244040aa0a..2baf3b583aed49 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts @@ -32,8 +32,7 @@ import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; -// FLAKY: https://github.com/elastic/kibana/issues/168284 -describe.skip('Endpoints page', { tags: ['@ess', '@serverless'] }, () => { +describe('Endpoints page', { tags: ['@ess', '@serverless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; let createdHost: CreateAndEnrollEndpointHostResponse; diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index 164afe89086d1b..fcead968801afa 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -332,16 +332,39 @@ export const dataLoadersForRealEndpoints = ( options: Omit ): Promise => { const { kbnClient, log } = await stackServicesPromise; - return createAndEnrollEndpointHost({ - useClosestVersionMatch: true, - ...options, - log, - kbnClient, - }).then((newHost) => { - return waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000).then(() => { + + let retryAttempt = 0; + const attemptCreateEndpointHost = async (): Promise => { + try { + log.info(`Creating endpoint host, attempt ${retryAttempt}`); + const newHost = await createAndEnrollEndpointHost({ + useClosestVersionMatch: true, + ...options, + log, + kbnClient, + }); + await waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000); return newHost; - }); - }); + } catch (err) { + log.info(`Caught error when setting up the agent: ${err}`); + if (retryAttempt === 0 && err.agentId) { + retryAttempt++; + await destroyEndpointHost(kbnClient, { + hostname: err.hostname || '', // No hostname in CI env for vagrant + agentId: err.agentId, + }); + log.info(`Deleted endpoint host ${err.agentId} and retrying`); + return attemptCreateEndpointHost(); + } else { + log.info( + `${retryAttempt} attempts of creating endpoint host failed, reason for the last failure was ${err}` + ); + throw err; + } + } + }; + + return attemptCreateEndpointHost(); }, destroyEndpointHost: async ( diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts index 7db2ad37317fdc..8b75c7e1b2af59 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts @@ -17,6 +17,6 @@ export const createEndpointHost = ( { agentPolicyId, }, - { timeout: timeout ?? 900000 } // 15 minutes, since setup can take 10 minutes and more. Task will time out if is not resolved within this time. + { timeout: timeout ?? 30 * 60 * 1000 } ); }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index ef38b120c38bbe..b716b9299ba023 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -234,7 +234,7 @@ const createMultipassVm = async ({ }; }; -const deleteMultipassVm = async (vmName: string): Promise => { +export const deleteMultipassVm = async (vmName: string): Promise => { if (process.env.CI) { await execa.command(`vagrant destroy -f`, { env: { @@ -339,7 +339,10 @@ const enrollHostWithFleet = async ({ ]); } log.info(`Waiting for Agent to check-in with Fleet`); - const agent = await waitForHostToEnroll(kbnClient, vmName, 240000); + + const agent = await waitForHostToEnroll(kbnClient, vmName, 8 * 60 * 1000); + + log.info(`Agent enrolled with Fleet, status: `, agent.status); return { agentId: agent.id, diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index a95b9f03caf2e9..e254b8d3a1570a 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -147,15 +147,19 @@ export const waitForHostToEnroll = async ( return elapsedTime > timeoutMs; }; let found: Agent | undefined; + let agentId: string | undefined; while (!found && !hasTimedOut()) { found = await retryOnError( async () => fetchFleetAgents(kbnClient, { perPage: 1, - kuery: `(local_metadata.host.hostname.keyword : "${hostname}") and (status:online)`, + kuery: `(local_metadata.host.hostname.keyword : "${hostname}")`, showInactive: false, - }).then((response) => response.items[0]), + }).then((response) => { + agentId = response.items[0]?.id; + return response.items.filter((agent) => agent.status === 'online')[0]; + }), RETRYABLE_TRANSIENT_ERRORS ); @@ -166,7 +170,14 @@ export const waitForHostToEnroll = async ( } if (!found) { - throw new Error(`Timed out waiting for host [${hostname}] to show up in Fleet`); + throw Object.assign( + new Error( + `Timed out waiting for host [${hostname}] to show up in Fleet in ${ + timeoutMs / 60 / 1000 + } seconds` + ), + { agentId, hostname } + ); } return found;