Skip to content

Commit

Permalink
fix: add filter, use GH name convention
Browse files Browse the repository at this point in the history
  • Loading branch information
Avishag Israeli committed Jan 28, 2021
1 parent 027c24a commit 229e12f
Showing 1 changed file with 39 additions and 37 deletions.
76 changes: 39 additions & 37 deletions test/cli-alert/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,46 +15,46 @@ const octokit = new Octokit({
});
const slackWebhook = new IncomingWebhook(SLACK_WEBHOOK_URL);

async function checkJobs(
smokeTest1: number,
smokeTest2: number,
async function discoverConsecutiveFailures(
firstWorkflowRun: number,
secondWorkflowRun: number,
): Promise<string[]> {
// Get all jobs of the latest 2 smoke tests
const jobs1 = (
const firstWorkflowRunJobs = (
await octokit.actions.listJobsForWorkflowRun({
owner: 'snyk',
repo: 'snyk',
// eslint-disable-next-line @typescript-eslint/camelcase
run_id: smokeTest1,
run_id: firstWorkflowRun,
})
).data.jobs;

const jobs2 = (
const secondWorkflowRunJobs = (
await octokit.actions.listJobsForWorkflowRun({
owner: 'snyk',
repo: 'snyk',
// eslint-disable-next-line @typescript-eslint/camelcase
run_id: smokeTest2,
run_id: secondWorkflowRun,
})
).data.jobs;

const failedJobs: string[] = [];

// If the same job failed in both smoke tests, it has been failing for 2 hours now. Save job to re-run later.
for (const jobName of jobs1.map((j) => j.name)) {
const job1 = jobs1.find((j) => j.name === jobName);
const job2 = jobs2.find((j) => j.name === jobName);
for (const jobName of firstWorkflowRunJobs.map((j) => j.name)) {
const firstJob = firstWorkflowRunJobs.find((j) => j.name === jobName);
const secondJob = secondWorkflowRunJobs.find((j) => j.name === jobName);

if (job1 === undefined || job2 === undefined) {
if (firstJob === undefined || secondJob === undefined) {
console.error(
`Could not find job ${jobName} in Smoke Tests ID: ${
job1 ? smokeTest2 : smokeTest1
firstJob ? secondWorkflowRun : firstWorkflowRun
}`,
);
process.exit(1);
} else if (
'failure' === job1.conclusion &&
job1?.conclusion === job2.conclusion
'failure' === firstJob.conclusion &&
firstJob?.conclusion === secondJob.conclusion
) {
console.log(`Found a job that failed 2 times in a row: ${jobName}`);
failedJobs.push(jobName);
Expand Down Expand Up @@ -113,7 +113,7 @@ async function checkJobConclusion(
failedJobs: string[],
): Promise<string[]> {
// Get conclusions of the jobs that failed before
const smokeTestsJobs = (
const workflowRunJobs = (
await octokit.actions.listJobsForWorkflowRun({
owner: 'snyk',
repo: 'snyk',
Expand All @@ -123,18 +123,18 @@ async function checkJobConclusion(
).data.jobs;

// Return false if jobs that failed before failed again
const rerunJobs = smokeTestsJobs.filter((job) =>
const rerunJobs = workflowRunJobs.filter((job) =>
failedJobs.includes(job.name),
);

const failedAgain: string[] = [];
const failedAgainJobs: string[] = [];
for (const job of rerunJobs) {
if (job.conclusion === 'failure') {
failedAgain.push(job.name);
failedAgainJobs.push(job.name);
}
}

return failedAgain;
return failedAgainJobs;
}

async function run() {
Expand All @@ -147,15 +147,17 @@ async function run() {
})
).data;

let smokeTestsID = 0;
for (const workflow of allWorkflows.workflows) {
if (workflow.name === 'Smoke Tests') {
smokeTestsID = workflow.id;
}
const smokeTestsID = allWorkflows.workflows.find(
(workflow) => workflow.name === 'Smoke Tests',
)?.id;

if (!smokeTestsID) {
console.error('Error: Could not find Smoke Tests workflow ID');
process.exit(1);
}

// Get latest smoke tests
const smokeTests = (
const workflowRuns = (
await octokit.actions.listWorkflowRuns({
owner: 'snyk',
repo: 'snyk',
Expand All @@ -167,26 +169,26 @@ async function run() {
console.log('Got latest smoke tests...');

// Check the latest 2 smoke tests for tests that had the same job fail 2 times in a row.
const latestSmokeTests = smokeTests.workflow_runs.slice(0, 2);
const latestWorkflowRuns = workflowRuns.workflow_runs.slice(0, 2);

console.log('Checking smoke tests jobs...');
const failedJobs = await checkJobs(
latestSmokeTests[0].id,
latestSmokeTests[1].id,
const failedWorkflows = await discoverConsecutiveFailures(
latestWorkflowRuns[0].id,
latestWorkflowRuns[1].id,
);

if (!failedJobs.length || failedJobs.length < 1) {
if (!failedWorkflows.length || failedWorkflows.length < 1) {
console.log(
'There were no 2 consecutive fails on a job. No need to alert.',
);
return;
}

console.log('Trying to re-run smoke test...');
const id = latestSmokeTests[0].id;
const id = latestWorkflowRuns[0].id;

// Check current status of smoke test and wait if it's still running
const smokeTest = (
// Check current status of smoke test workflow and wait if it's still running
const latestRun = (
await octokit.actions.getWorkflowRun({
owner: 'snyk',
repo: 'snyk',
Expand All @@ -195,7 +197,7 @@ async function run() {
})
).data;

if (smokeTest.status !== 'completed') {
if (latestRun.status !== 'completed') {
console.log('First wait for current run to finish...');
await waitForConclusion(id);
}
Expand All @@ -211,12 +213,12 @@ async function run() {

// Wait for run to finish
await waitForConclusion(id);
const failedAgain = await checkJobConclusion(id, failedJobs);
const failedAgainJobs = await checkJobConclusion(id, failedWorkflows);
console.log('Re-run completed.');

// If run failed again, send Slack alert
failedAgain.length > 0
? await sendSlackAlert(failedAgain)
failedAgainJobs.length > 0
? await sendSlackAlert(failedAgainJobs)
: console.log('Jobs succeeded after re-run. Do not alert.');
} catch (error) {
console.error(error);
Expand Down

0 comments on commit 229e12f

Please sign in to comment.