Skip to content

Commit

Permalink
fix: Lambda image not updating while stack is deployed (#116)
Browse files Browse the repository at this point in the history
Use a simpler method to avoid conflicts when updating the function code after the image is updated. Instead of looking for a running stack operation, just try again in a loop every 10 seconds. The current code assumed some special handling in CloudFormation that would update the function later. It wasn't working. I believe it was broken by #103.
  • Loading branch information
kichik committed Oct 9, 2022
1 parent 2acd890 commit 3a955b6
Show file tree
Hide file tree
Showing 10 changed files with 707 additions and 97 deletions.
4 changes: 4 additions & 0 deletions .projen/deps.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions .projenrc.js
Expand Up @@ -20,6 +20,7 @@ const project = new awscdk.AwsCdkConstructLibrary({
'@octokit/rest',
'aws-sdk',
'@aws-sdk/types',
'@aws-sdk/client-lambda',
'@types/aws-lambda',
'semver',
'@types/semver',
Expand Down
1 change: 1 addition & 0 deletions package.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

43 changes: 23 additions & 20 deletions src/lambdas/update-lambda/index.ts
@@ -1,36 +1,39 @@
/* eslint-disable-next-line import/no-extraneous-dependencies */
import { ResourceConflictException } from '@aws-sdk/client-lambda';
/* eslint-disable-next-line import/no-extraneous-dependencies */
import * as AWS from 'aws-sdk';

const cfn = new AWS.CloudFormation();
const lambda = new AWS.Lambda();

interface Input {
readonly lambdaName: string;
readonly repositoryUri: string;
readonly repositoryTag: string;
readonly stackName: string;
}

function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}

export async function handler(event: Input) {
console.log(JSON.stringify(event));

const stacks = await cfn.describeStacks({
StackName: event.stackName,
}).promise();

if (stacks.Stacks?.length != 1) {
console.error(`Unable to find stack ${event.stackName}`);
return;
while (true) {
try {
await lambda.updateFunctionCode({
FunctionName: event.lambdaName,
ImageUri: `${event.repositoryUri}:${event.repositoryTag}`,
Publish: true,
}).promise();
break;
} catch (e) {
if (e instanceof ResourceConflictException) {
// keep trying if function is already being updated by CloudFormation
// this can happen if we update some settings on the function and the image code at the same time
await sleep(10000);
} else {
throw e;
}
}
}

if (stacks.Stacks[0].StackStatus.endsWith('_IN_PROGRESS')) {
console.error(`Stack ${event.stackName} is already in progress, skipping Lambda update as the stack will do it for us. If we update here, it may conflict with the stack operation.`);
return;
}

await lambda.updateFunctionCode({
FunctionName: event.lambdaName,
ImageUri: `${event.repositoryUri}:${event.repositoryTag}`,
Publish: true,
}).promise();
}
15 changes: 1 addition & 14 deletions src/providers/lambda.ts
Expand Up @@ -240,21 +240,9 @@ export class LambdaRunner extends Construct implements IRunnerProvider {
// Lambda needs to be pointing to a specific image digest and not just a tag.
// Whenever we update the tag to a new digest, we need to update the lambda.

let stack = cdk.Stack.of(this);

const updater = BundledNodejsFunction.singleton(this, 'update-lambda', {
description: 'Function that updates a GitHub Actions runner function with the latest image digest after the image has been rebuilt',
timeout: cdk.Duration.seconds(30),
initialPolicy: [
new iam.PolicyStatement({
actions: ['cloudformation:DescribeStacks'],
resources: [stack.formatArn({
service: 'cloudformation',
resource: 'stack',
resourceName: `${stack.stackName}/*`,
})],
}),
],
timeout: cdk.Duration.minutes(15),
});

updater.addToRolePolicy(new iam.PolicyStatement({
Expand All @@ -267,7 +255,6 @@ export class LambdaRunner extends Construct implements IRunnerProvider {
lambdaName: this.function.functionName,
repositoryUri: image.imageRepository.repositoryUri,
repositoryTag: image.imageTag,
stackName: stack.stackName,
}),
});

Expand Down
10 changes: 5 additions & 5 deletions test/default.integ.snapshot/github-runners-test.assets.json
Expand Up @@ -118,15 +118,15 @@
}
}
},
"35c99ca05f12b61868c715d657cd142b535de141a93e018fd30f8198753d147e": {
"36e017cd2150811999eeb9781677a0b3e91f24239b778c65b24e2578900b5795": {
"source": {
"path": "asset.35c99ca05f12b61868c715d657cd142b535de141a93e018fd30f8198753d147e",
"path": "asset.36e017cd2150811999eeb9781677a0b3e91f24239b778c65b24e2578900b5795",
"packaging": "zip"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "35c99ca05f12b61868c715d657cd142b535de141a93e018fd30f8198753d147e.zip",
"objectKey": "36e017cd2150811999eeb9781677a0b3e91f24239b778c65b24e2578900b5795.zip",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
Expand Down Expand Up @@ -209,15 +209,15 @@
}
}
},
"5271f4eea59a216248e03b16056e9dc13dfacd5794f43be791846fd8280ac885": {
"2664f401baade08480f8dec055d75706a80425ac9288f3e4e9f8db0bf4899289": {
"source": {
"path": "github-runners-test.template.json",
"packaging": "file"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "5271f4eea59a216248e03b16056e9dc13dfacd5794f43be791846fd8280ac885.json",
"objectKey": "2664f401baade08480f8dec055d75706a80425ac9288f3e4e9f8db0bf4899289.json",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
Expand Down
32 changes: 4 additions & 28 deletions test/default.integ.snapshot/github-runners-test.template.json
Expand Up @@ -1773,7 +1773,7 @@
"Name"
]
},
"\",\"repositoryTag\":\"latest\",\"stackName\":\"github-runners-test\"}"
"\",\"repositoryTag\":\"latest\"}"
]
]
}
Expand Down Expand Up @@ -4744,30 +4744,6 @@
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Action": "cloudformation:DescribeStacks",
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":cloudformation:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":stack/github-runners-test/*"
]
]
}
},
{
"Action": "lambda:UpdateFunctionCode",
"Effect": "Allow",
Expand Down Expand Up @@ -4806,7 +4782,7 @@
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
},
"S3Key": "35c99ca05f12b61868c715d657cd142b535de141a93e018fd30f8198753d147e.zip"
"S3Key": "36e017cd2150811999eeb9781677a0b3e91f24239b778c65b24e2578900b5795.zip"
},
"Role": {
"Fn::GetAtt": [
Expand All @@ -4822,7 +4798,7 @@
},
"Handler": "index.handler",
"Runtime": "nodejs14.x",
"Timeout": 30
"Timeout": 900
},
"DependsOn": [
"updatelambdadcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicy1EF644A6",
Expand Down Expand Up @@ -5436,7 +5412,7 @@
"Name"
]
},
"\",\"repositoryTag\":\"latest\",\"stackName\":\"github-runners-test\"}"
"\",\"repositoryTag\":\"latest\"}"
]
]
}
Expand Down
2 changes: 1 addition & 1 deletion test/default.integ.snapshot/manifest.json
Expand Up @@ -23,7 +23,7 @@
"validateOnSynth": false,
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
"cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5271f4eea59a216248e03b16056e9dc13dfacd5794f43be791846fd8280ac885.json",
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/2664f401baade08480f8dec055d75706a80425ac9288f3e4e9f8db0bf4899289.json",
"requiresBootstrapStackVersion": 6,
"bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
"additionalDependencies": [
Expand Down
32 changes: 4 additions & 28 deletions test/default.integ.snapshot/tree.json
Expand Up @@ -2513,7 +2513,7 @@
"Name"
]
},
"\",\"repositoryTag\":\"latest\",\"stackName\":\"github-runners-test\"}"
"\",\"repositoryTag\":\"latest\"}"
]
]
}
Expand Down Expand Up @@ -6562,30 +6562,6 @@
"aws:cdk:cloudformation:props": {
"policyDocument": {
"Statement": [
{
"Action": "cloudformation:DescribeStacks",
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":cloudformation:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":stack/github-runners-test/*"
]
]
}
},
{
"Action": "lambda:UpdateFunctionCode",
"Effect": "Allow",
Expand Down Expand Up @@ -6670,7 +6646,7 @@
"s3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
},
"s3Key": "35c99ca05f12b61868c715d657cd142b535de141a93e018fd30f8198753d147e.zip"
"s3Key": "36e017cd2150811999eeb9781677a0b3e91f24239b778c65b24e2578900b5795.zip"
},
"role": {
"Fn::GetAtt": [
Expand All @@ -6686,7 +6662,7 @@
},
"handler": "index.handler",
"runtime": "nodejs14.x",
"timeout": 30
"timeout": 900
}
},
"constructInfo": {
Expand Down Expand Up @@ -7501,7 +7477,7 @@
"Name"
]
},
"\",\"repositoryTag\":\"latest\",\"stackName\":\"github-runners-test\"}"
"\",\"repositoryTag\":\"latest\"}"
]
]
}
Expand Down

0 comments on commit 3a955b6

Please sign in to comment.