Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feat][sc-45108] Improve logging and handle delete-during-wait edge case #1

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
64 changes: 49 additions & 15 deletions src/cloudformation.ts
@@ -1,7 +1,14 @@
/* eslint-disable no-await-in-loop */
import { CloudFormation, Stack, StackEvent } from '@aws-sdk/client-cloudformation';
import { sleep } from './async.js';
import { logInfo, logSuccess } from './log.js';
import {
logError,
LogFn,
logInfo,
logSuccess,
logSuccessWithTimestamp,
logWithTimestamp,
} from './log.js';

export function createCloudFormationClient({ region }: { region: string }) {
return new CloudFormation({ region });
Expand All @@ -15,6 +22,30 @@ function isCloudFormationStatusReadyForUpdate(status: string) {
);
}

function logEvent(event: StackEvent) {
const {
ResourceStatus: status = 'Unknown Status',
ResourceStatusReason: reason = null,
LogicalResourceId: logicalResourceId = 'Unknown Logical ID',
ResourceType: resourceType = 'Unknown Type',
Timestamp: timestamp,
} = event;
const reasonSuffix = reason ? ` (${reason})` : '';
let logFunction: LogFn;
if (status.endsWith('_FAILED')) {
logFunction = logError;
} else if (status.endsWith('_COMPLETE')) {
logFunction = logSuccess;
} else {
logFunction = logInfo;
}
logWithTimestamp(
[`${status}${reasonSuffix}`, `${logicalResourceId} (${resourceType})`],
logFunction,
timestamp,
);
}

export async function waitForStackToBeReadyForUpdate({
cloudFormation,
stackName,
Expand All @@ -40,37 +71,40 @@ export async function waitForStackToBeReadyForUpdate({
[stack] = stacks;
} catch (error) {
if ((error as { Code: string }).Code === 'ValidationError') {
logInfo('Stack does not exist. Safe to create.');
logSuccessWithTimestamp('Stack does not exist. Safe to create.');
return;
}
throw error;
}
if (!stack) {
logSuccess('Stack does not exist. Safe to create.');
logSuccessWithTimestamp('Stack does not exist. Safe to create.');
}
const { StackStatus: stackStatus } = stack;
if (!stackStatus) {
throw new Error('StackStatus is undefined');
}
if (isCloudFormationStatusReadyForUpdate(stackStatus)) {
logSuccess('Stack is ready for update.');
logSuccessWithTimestamp('Stack is ready for update.');
return;
}

let latestEventId = '';
let latestStatus = '';

do {
const events = await describeStackEventsSince(latestEventId);
for (const event of events) {
const { EventId: eventId, ResourceStatusReason: reason, ResourceStatus: status } = event;
const logPrefix = `[${event.Timestamp?.toISOString() || '(no timestamp)'}]`;
if (status) {
logInfo(`${logPrefix} Status: ${status}`);
}
if (reason) {
logInfo(`${logPrefix} Reason: ${reason}`);
let events: StackEvent[] = [];
try {
events = await describeStackEventsSince(latestEventId);
} catch (error) {
if ((error as { Code: string }).Code === 'ValidationError') {
logSuccessWithTimestamp('Stack no longer exists (e.g. was deleted). Safe to create.');
return;
}
throw error;
}
for (const event of events) {
const { EventId: eventId, ResourceStatus: status } = event;
logEvent(event);
if (status && isStackEventForThisStack(event)) {
latestStatus = status;
}
Expand All @@ -80,10 +114,10 @@ export async function waitForStackToBeReadyForUpdate({
}

if (!isCloudFormationStatusReadyForUpdate(latestStatus)) {
logInfo('Stack not yet ready. Waiting 10 seconds...');
logWithTimestamp('Stack not yet ready. Waiting 10 seconds...');
await sleep(10000);
}
} while (!isCloudFormationStatusReadyForUpdate(latestStatus));

logSuccess('Stack is ready for update.');
logSuccessWithTimestamp('Stack is ready for update.');
}
40 changes: 40 additions & 0 deletions src/log.ts
@@ -1,6 +1,8 @@
import * as util from 'node:util';
import chalk from 'chalk';

export type LogFn = (...formatArgs: Parameters<typeof util.format>) => void;

export function logSuccess(...formatArgs: Parameters<typeof util.format>) {
process.stdout.write(`${chalk.green(util.format(...formatArgs))}\n`);
}
Expand All @@ -12,3 +14,41 @@ export function logError(...formatArgs: Parameters<typeof util.format>) {
export function logInfo(...formatArgs: Parameters<typeof util.format>) {
process.stdout.write(`${chalk.blue(util.format(...formatArgs))}\n`);
}

export function logWithTimestamp(
logLineOrLines: (Parameters<typeof util.format> | string)[] | string,
logFunction: LogFn = logInfo,
dateTime: Date = new Date(),
) {
const timestamp = dateTime.toISOString();
const logLines = Array.isArray(logLineOrLines) ? logLineOrLines : [logLineOrLines];
const processedLines = logLines.map((line) =>
typeof line === 'string' ? line : util.format(...line),
);
const [firstLine, ...middleLines] = processedLines;
const firstLineSeparator = middleLines.length > 0 ? '⎡' : '[';
const lastLine = middleLines.pop();
const timestampPrefix = `| ${timestamp} `;
logFunction(`${timestampPrefix}${firstLineSeparator} ${firstLine}`);
const padding = ' '.repeat(timestampPrefix.length - 1);
for (const line of middleLines) {
logFunction(`|${padding}| ${line})`);
}
if (lastLine) {
logFunction(`|${padding}⎣ ${lastLine}`);
}
}

export function logSuccessWithTimestamp(
logLineOrLines: (Parameters<typeof util.format> | string)[] | string,
dateTime: Date = new Date(),
) {
logWithTimestamp(logLineOrLines, logSuccess, dateTime);
}

export function logErrorWithTimestamp(
logLineOrLines: (Parameters<typeof util.format> | string)[] | string,
dateTime: Date = new Date(),
) {
logWithTimestamp(logLineOrLines, logError, dateTime);
}