Skip to content

Commit

Permalink
Improve retry logic in Cypress CI (#183488)
Browse files Browse the repository at this point in the history 
## Summary

Previously when Fleet server setup step has failed the whole Cypress run
was aborted.
Added try catch on all the steps, so in case any of them fails we retry
whole spec again
  • Loading branch information
@patrykkopycinski
patrykkopycinski committed May 17, 2024
1 parent 012d62f commit d87d04c
Show file tree
Hide file tree
Showing 2 changed files with 106 additions and 94 deletions.
8 changes: 7 additions & 1 deletion ...k/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import type { KbnClient } from '@kbn/test';
import execa from 'execa';
import chalk from 'chalk';
import assert from 'assert';
import pRetry from 'p-retry';
import type { AgentPolicy, CreateAgentPolicyResponse, Output } from '@kbn/fleet-plugin/common';
import {
AGENT_POLICY_API_ROUTES,
Expand Down Expand Up @@ -137,7 +138,12 @@ export const startFleetServer = async ({
const isServerless = await isServerlessKibanaFlavor(kbnClient);
const policyId =
policy || !isServerless ? await getOrCreateFleetServerAgentPolicyId(kbnClient, logger) : '';
const serviceToken = isServerless ? '' : await generateFleetServiceToken(kbnClient, logger);
const serviceToken = isServerless
? ''
: await pRetry(async () => generateFleetServiceToken(kbnClient, logger), {
retries: 2,
forever: false,
});
const startedFleetServer = await startFleetServerWithDocker({
kbnClient,
logger,
Expand Down
192 changes: 99 additions & 93 deletions x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,100 +316,107 @@ ${JSON.stringify(
ci: process.env.CI,
};

const shutdownEs = await pRetry(
async () =>
runElasticsearch({
config,
log,
name: `ftr-${esPort}`,
esFrom: config.get('esTestCluster')?.from || 'snapshot',
onEarlyExit,
}),
{ retries: 2, forever: false }
);

await runKibanaServer({
procs,
config,
installDir: options?.installDir,
extraKbnOpts:
options?.installDir || options?.ci || !isOpen
? []
: ['--dev', '--no-dev-config', '--no-dev-credentials'],
onEarlyExit,
inspect: argv.inspect,
});

// Setup fleet if Cypress config requires it
let fleetServer: void | StartedFleetServer;
if (cypressConfigFile.env?.WITH_FLEET_SERVER) {
log.info(`Setting up fleet-server for this Cypress config`);

const kbnClient = createKbnClient({
url: baseUrl,
username: config.get('servers.kibana.username'),
password: config.get('servers.kibana.password'),
log,
let shutdownEs;

try {
shutdownEs = await pRetry(
async () =>
runElasticsearch({
config,
log,
name: `ftr-${esPort}`,
esFrom: config.get('esTestCluster')?.from || 'snapshot',
onEarlyExit,
}),
{ retries: 2, forever: false }
);

await runKibanaServer({
procs,
config,
installDir: options?.installDir,
extraKbnOpts:
options?.installDir || options?.ci || !isOpen
? []
: ['--dev', '--no-dev-config', '--no-dev-credentials'],
onEarlyExit,
inspect: argv.inspect,
});

fleetServer = await startFleetServer({
kbnClient,
logger: log,
port:
fleetServerPort ?? config.has('servers.fleetserver.port')
? (config.get('servers.fleetserver.port') as number)
: undefined,
// `force` is needed to ensure that any currently running fleet server (perhaps left
// over from an interrupted run) is killed and a new one restarted
force: true,
});
}
if (cypressConfigFile.env?.WITH_FLEET_SERVER) {
log.info(`Setting up fleet-server for this Cypress config`);

await providers.loadAll();
const kbnClient = createKbnClient({
url: baseUrl,
username: config.get('servers.kibana.username'),
password: config.get('servers.kibana.password'),
log,
});

const functionalTestRunner = new FunctionalTestRunner(
log,
config,
EsVersion.getDefault()
);
fleetServer = await pRetry(
async () =>
startFleetServer({
kbnClient,
logger: log,
port:
fleetServerPort ?? config.has('servers.fleetserver.port')
? (config.get('servers.fleetserver.port') as number)
: undefined,
// `force` is needed to ensure that any currently running fleet server (perhaps left
// over from an interrupted run) is killed and a new one restarted
force: true,
}),
{ retries: 2, forever: false }
);
}

const ftrEnv = await pRetry(() => functionalTestRunner.run(abortCtrl.signal), {
retries: 1,
});
await providers.loadAll();

log.debug(
`Env. variables returned by [functionalTestRunner.run()]:\n`,
JSON.stringify(ftrEnv, null, 2)
);
const functionalTestRunner = new FunctionalTestRunner(
log,
config,
EsVersion.getDefault()
);

// Normalized the set of available env vars in cypress
const cyCustomEnv = {
...ftrEnv,
const ftrEnv = await pRetry(() => functionalTestRunner.run(abortCtrl.signal), {
retries: 1,
});

// NOTE:
// ELASTICSEARCH_URL needs to be created here with auth because SIEM cypress setup depends on it. At some
// points we should probably try to refactor that code to use `ELASTICSEARCH_URL_WITH_AUTH` instead
ELASTICSEARCH_URL:
ftrEnv.ELASTICSEARCH_URL ?? createUrlFromFtrConfig('elasticsearch', true),
ELASTICSEARCH_URL_WITH_AUTH: createUrlFromFtrConfig('elasticsearch', true),
ELASTICSEARCH_USERNAME:
ftrEnv.ELASTICSEARCH_USERNAME ?? config.get('servers.elasticsearch.username'),
ELASTICSEARCH_PASSWORD:
ftrEnv.ELASTICSEARCH_PASSWORD ?? config.get('servers.elasticsearch.password'),
log.debug(
`Env. variables returned by [functionalTestRunner.run()]:\n`,
JSON.stringify(ftrEnv, null, 2)
);

FLEET_SERVER_URL: createUrlFromFtrConfig('fleetserver'),
// Normalized the set of available env vars in cypress
const cyCustomEnv = {
...ftrEnv,

KIBANA_URL: baseUrl,
KIBANA_URL_WITH_AUTH: createUrlFromFtrConfig('kibana', true),
KIBANA_USERNAME: config.get('servers.kibana.username'),
KIBANA_PASSWORD: config.get('servers.kibana.password'),
// NOTE:
// ELASTICSEARCH_URL needs to be created here with auth because SIEM cypress setup depends on it. At some
// points we should probably try to refactor that code to use `ELASTICSEARCH_URL_WITH_AUTH` instead
ELASTICSEARCH_URL:
ftrEnv.ELASTICSEARCH_URL ?? createUrlFromFtrConfig('elasticsearch', true),
ELASTICSEARCH_URL_WITH_AUTH: createUrlFromFtrConfig('elasticsearch', true),
ELASTICSEARCH_USERNAME:
ftrEnv.ELASTICSEARCH_USERNAME ?? config.get('servers.elasticsearch.username'),
ELASTICSEARCH_PASSWORD:
ftrEnv.ELASTICSEARCH_PASSWORD ?? config.get('servers.elasticsearch.password'),

IS_SERVERLESS: config.get('serverless'),
FLEET_SERVER_URL: createUrlFromFtrConfig('fleetserver'),

...argv.env,
};
KIBANA_URL: baseUrl,
KIBANA_URL_WITH_AUTH: createUrlFromFtrConfig('kibana', true),
KIBANA_USERNAME: config.get('servers.kibana.username'),
KIBANA_PASSWORD: config.get('servers.kibana.password'),

log.info(`
IS_SERVERLESS: config.get('serverless'),

...argv.env,
};

log.info(`
----------------------------------------------
Cypress run ENV for file: ${filePath}:
----------------------------------------------
Expand All @@ -419,18 +426,17 @@ ${JSON.stringify(cyCustomEnv, null, 2)}
----------------------------------------------
`);

if (isOpen) {
await cypress.open({
configFile: cypressConfigFilePath,
config: {
e2e: {
baseUrl,
if (isOpen) {
await cypress.open({
configFile: cypressConfigFilePath,
config: {
e2e: {
baseUrl,
},
env: cyCustomEnv,
},
env: cyCustomEnv,
},
});
} else {
try {
});
} else {
result = await cypress.run({
browser: 'chrome',
spec: filePath,
Expand All @@ -450,17 +456,17 @@ ${JSON.stringify(cyCustomEnv, null, 2)}
if (!(result as CypressCommandLine.CypressRunResult)?.totalFailed) {
_.pull(failedSpecFilePaths, filePath);
}
} catch (error) {
result = error;
}
} catch (error) {
log.error(error);
}

if (fleetServer) {
await fleetServer.stop();
}

await procs.stop('kibana');
await shutdownEs();
await shutdownEs?.();
cleanupServerPorts({ esPort, kibanaPort, fleetServerPort });

return result;
Expand Down

0 comments on commit d87d04c

Please sign in to comment.