Skip to content

Commit 2ce4a87

Browse files
CaerusKarumergify[bot]
authored andcommitted
feat(s3-deployment): allow multiple Sources for single Deployment (#4105)
* feat(s3-deployment): allow multiple Sources for single Deployment In some cases, a user may want to inject other sources into a deployment. For instance, storing JavaScript bundles in one directory and images in another, then combining them in a single bucket for CloudFront distributions. * All sources will be fetched and consolidated into one directory before being zipped as normal prior to deployment BREAKING CHANGE: * Property `source` is now `sources` and is a `Source` array * fixup! feat(s3-deployment): allow multiple Sources for single Deployment * fixup! feat(s3-deployment): allow multiple Sources for single Deployment * fixup! feat(s3-deployment): allow multiple Sources for single Deployment
1 parent 57a7ae0 commit 2ce4a87

13 files changed

+264
-134
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ coverage/
1919
.LAST_BUILD
2020
*.sw[a-z]
2121
*~
22+
.idea
2223

2324
# We don't want tsconfig at the root
2425
/tsconfig.json

packages/@aws-cdk/aws-s3-deployment/README.md

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717

1818
> __Status: Experimental__
1919
20-
This library allows populating an S3 bucket with the contents of a .zip file
21-
from another S3 bucket or from local disk.
20+
This library allows populating an S3 bucket with the contents of .zip files
21+
from other S3 buckets or from local disk.
2222

2323
The following example defines a publicly accessible S3 bucket with web hosting
2424
enabled and populates it from a local directory on disk.
@@ -30,7 +30,7 @@ const websiteBucket = new s3.Bucket(this, 'WebsiteBucket', {
3030
});
3131

3232
new s3deploy.BucketDeployment(this, 'DeployWebsite', {
33-
source: s3deploy.Source.asset('./website-dist'),
33+
sources: [s3deploy.Source.asset('./website-dist')],
3434
destinationBucket: websiteBucket,
3535
destinationKeyPrefix: 'web/static' // optional prefix in destination bucket
3636
});
@@ -40,13 +40,15 @@ This is what happens under the hood:
4040

4141
1. When this stack is deployed (either via `cdk deploy` or via CI/CD), the
4242
contents of the local `website-dist` directory will be archived and uploaded
43-
to an intermediary assets bucket.
43+
to an intermediary assets bucket. If there is more than one source, they will
44+
be individually uploaded.
4445
2. The `BucketDeployment` construct synthesizes a custom CloudFormation resource
4546
of type `Custom::CDKBucketDeployment` into the template. The source bucket/key
4647
is set to point to the assets bucket.
4748
3. The custom resource downloads the .zip archive, extracts it and issues `aws
4849
s3 sync --delete` against the destination bucket (in this case
49-
`websiteBucket`).
50+
`websiteBucket`). If there is more than one source, the sources will be
51+
downloaded and merged pre-deployment at this step.
5052

5153
## Supported sources
5254

@@ -82,7 +84,7 @@ const distribution = new cloudfront.CloudFrontWebDistribution(this, 'Distributio
8284
});
8385

8486
new s3deploy.BucketDeployment(this, 'DeployWithInvalidation', {
85-
source: s3deploy.Source.asset('./website-dist'),
87+
sources: [s3deploy.Source.asset('./website-dist')],
8688
destinationBucket: bucket,
8789
distribution,
8890
distributionPaths: ['/images/*.png'],

packages/@aws-cdk/aws-s3-deployment/lambda/src/index.py

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ def cfn_error(message=None):
3939
physical_id = event.get('PhysicalResourceId', None)
4040

4141
try:
42-
source_bucket_name = props['SourceBucketName']
43-
source_object_key = props['SourceObjectKey']
42+
source_bucket_names = props['SourceBucketNames']
43+
source_object_keys = props['SourceObjectKeys']
4444
dest_bucket_name = props['DestinationBucketName']
4545
dest_bucket_prefix = props.get('DestinationBucketKeyPrefix', '')
4646
retain_on_delete = props.get('RetainOnDelete', "true") == "true"
@@ -62,7 +62,7 @@ def cfn_error(message=None):
6262
if dest_bucket_prefix == "/":
6363
dest_bucket_prefix = ""
6464

65-
s3_source_zip = "s3://%s/%s" % (source_bucket_name, source_object_key)
65+
s3_source_zips = map(lambda name, key: "s3://%s/%s" % (name, key), source_bucket_names, source_object_keys)
6666
s3_dest = "s3://%s/%s" % (dest_bucket_name, dest_bucket_prefix)
6767

6868
old_s3_dest = "s3://%s/%s" % (old_props.get("DestinationBucketName", ""), old_props.get("DestinationBucketKeyPrefix", ""))
@@ -96,7 +96,7 @@ def cfn_error(message=None):
9696
aws_command("s3", "rm", old_s3_dest, "--recursive")
9797

9898
if request_type == "Update" or request_type == "Create":
99-
s3_deploy(s3_source_zip, s3_dest)
99+
s3_deploy(s3_source_zips, s3_dest)
100100

101101
if distribution_id:
102102
cloudfront_invalidate(distribution_id, distribution_paths)
@@ -109,8 +109,8 @@ def cfn_error(message=None):
109109
cfn_error(str(e))
110110

111111
#---------------------------------------------------------------------------------------------------
112-
# populate all files from s3_source_zip to a destination bucket
113-
def s3_deploy(s3_source_zip, s3_dest):
112+
# populate all files from s3_source_zips to a destination bucket
113+
def s3_deploy(s3_source_zips, s3_dest):
114114
# create a temporary working directory
115115
workdir=tempfile.mkdtemp()
116116
logger.info("| workdir: %s" % workdir)
@@ -120,12 +120,13 @@ def s3_deploy(s3_source_zip, s3_dest):
120120
os.mkdir(contents_dir)
121121

122122
# download the archive from the source and extract to "contents"
123-
archive=os.path.join(workdir, 'archive.zip')
124-
logger.info("| archive: %s" % archive)
125-
aws_command("s3", "cp", s3_source_zip, archive)
126-
logger.info("| extracting archive to: %s" % contents_dir)
127-
with ZipFile(archive, "r") as zip:
128-
zip.extractall(contents_dir)
123+
for s3_source_zip in s3_source_zips:
124+
archive=os.path.join(workdir, str(uuid4()))
125+
logger.info("archive: %s" % archive)
126+
aws_command("s3", "cp", s3_source_zip, archive)
127+
logger.info("| extracting archive to: %s\n" % contents_dir)
128+
with ZipFile(archive, "r") as zip:
129+
zip.extractall(contents_dir)
129130

130131
# sync from "contents" to destination
131132
aws_command("s3", "sync", "--delete", contents_dir, s3_dest)

packages/@aws-cdk/aws-s3-deployment/lambda/test/test.py

Lines changed: 60 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ def setUp(self):
2323

2424
def test_invalid_request(self):
2525
resp = invoke_handler("Create", {}, expected_status="FAILED")
26-
self.assertEqual(resp["Reason"], "missing request resource property 'SourceBucketName'. props: {}")
26+
self.assertEqual(resp["Reason"], "missing request resource property 'SourceBucketNames'. props: {}")
2727

2828
def test_create_update(self):
2929
invoke_handler("Create", {
30-
"SourceBucketName": "<source-bucket>",
31-
"SourceObjectKey": "<source-object-key>",
30+
"SourceBucketNames": ["<source-bucket>"],
31+
"SourceObjectKeys": ["<source-object-key>"],
3232
"DestinationBucketName": "<dest-bucket-name>"
3333
})
3434

@@ -37,10 +37,25 @@ def test_create_update(self):
3737
"s3 sync --delete contents.zip s3://<dest-bucket-name>/"
3838
)
3939

40+
def test_create_update_multiple_sources(self):
41+
invoke_handler("Create", {
42+
"SourceBucketNames": ["<source-bucket1>", "<source-bucket2>"],
43+
"SourceObjectKeys": ["<source-object-key1>", "<source-object-key2>"],
44+
"DestinationBucketName": "<dest-bucket-name>"
45+
})
46+
47+
# Note: these are different files in real-life. For testing purposes, we hijack
48+
# the command to output a static filename, archive.zip
49+
self.assertAwsCommands(
50+
"s3 cp s3://<source-bucket1>/<source-object-key1> archive.zip",
51+
"s3 cp s3://<source-bucket2>/<source-object-key2> archive.zip",
52+
"s3 sync --delete contents.zip s3://<dest-bucket-name>/"
53+
)
54+
4055
def test_create_with_backslash_prefix_same_as_no_prefix(self):
4156
invoke_handler("Create", {
42-
"SourceBucketName": "<source-bucket>",
43-
"SourceObjectKey": "<source-object-key>",
57+
"SourceBucketNames": ["<source-bucket>"],
58+
"SourceObjectKeys": ["<source-object-key>"],
4459
"DestinationBucketName": "<dest-bucket-name>",
4560
"DestinationBucketKeyPrefix": "/"
4661
})
@@ -53,8 +68,8 @@ def test_create_with_backslash_prefix_same_as_no_prefix(self):
5368

5469
def test_create_update_with_dest_key(self):
5570
invoke_handler("Create", {
56-
"SourceBucketName": "<source-bucket>",
57-
"SourceObjectKey": "<source-object-key>",
71+
"SourceBucketNames": ["<source-bucket>"],
72+
"SourceObjectKeys": ["<source-object-key>"],
5873
"DestinationBucketName": "<dest-bucket-name>",
5974
"DestinationBucketKeyPrefix": "<dest-key-prefix>"
6075
})
@@ -66,8 +81,8 @@ def test_create_update_with_dest_key(self):
6681

6782
def test_delete_no_retain(self):
6883
invoke_handler("Delete", {
69-
"SourceBucketName": "<source-bucket>",
70-
"SourceObjectKey": "<source-object-key>",
84+
"SourceBucketNames": ["<source-bucket>"],
85+
"SourceObjectKeys": ["<source-object-key>"],
7186
"DestinationBucketName": "<dest-bucket-name>",
7287
"RetainOnDelete": "false"
7388
}, physical_id="<physicalid>")
@@ -76,8 +91,8 @@ def test_delete_no_retain(self):
7691

7792
def test_delete_with_dest_key(self):
7893
invoke_handler("Delete", {
79-
"SourceBucketName": "<source-bucket>",
80-
"SourceObjectKey": "<source-object-key>",
94+
"SourceBucketNames": ["<source-bucket>"],
95+
"SourceObjectKeys": ["<source-object-key>"],
8196
"DestinationBucketName": "<dest-bucket-name>",
8297
"DestinationBucketKeyPrefix": "<dest-key-prefix>",
8398
"RetainOnDelete": "false"
@@ -87,8 +102,8 @@ def test_delete_with_dest_key(self):
87102

88103
def test_delete_with_retain_explicit(self):
89104
invoke_handler("Delete", {
90-
"SourceBucketName": "<source-bucket>",
91-
"SourceObjectKey": "<source-object-key>",
105+
"SourceBucketNames": ["<source-bucket>"],
106+
"SourceObjectKeys": ["<source-object-key>"],
92107
"DestinationBucketName": "<dest-bucket-name>",
93108
"RetainOnDelete": "true"
94109
}, physical_id="<physicalid>")
@@ -99,8 +114,8 @@ def test_delete_with_retain_explicit(self):
99114
# RetainOnDelete=true is the default
100115
def test_delete_with_retain_implicit_default(self):
101116
invoke_handler("Delete", {
102-
"SourceBucketName": "<source-bucket>",
103-
"SourceObjectKey": "<source-object-key>",
117+
"SourceBucketNames": ["<source-bucket>"],
118+
"SourceObjectKeys": ["<source-object-key>"],
104119
"DestinationBucketName": "<dest-bucket-name>"
105120
}, physical_id="<physicalid>")
106121

@@ -109,8 +124,8 @@ def test_delete_with_retain_implicit_default(self):
109124

110125
def test_delete_with_retain_explicitly_false(self):
111126
invoke_handler("Delete", {
112-
"SourceBucketName": "<source-bucket>",
113-
"SourceObjectKey": "<source-object-key>",
127+
"SourceBucketNames": ["<source-bucket>"],
128+
"SourceObjectKeys": ["<source-object-key>"],
114129
"DestinationBucketName": "<dest-bucket-name>",
115130
"RetainOnDelete": "false"
116131
}, physical_id="<physicalid>")
@@ -125,8 +140,8 @@ def test_delete_with_retain_explicitly_false(self):
125140

126141
def test_update_same_dest(self):
127142
invoke_handler("Update", {
128-
"SourceBucketName": "<source-bucket>",
129-
"SourceObjectKey": "<source-object-key>",
143+
"SourceBucketNames": ["<source-bucket>"],
144+
"SourceObjectKeys": ["<source-object-key>"],
130145
"DestinationBucketName": "<dest-bucket-name>",
131146
}, old_resource_props={
132147
"DestinationBucketName": "<dest-bucket-name>",
@@ -150,8 +165,8 @@ def mock_make_api_call(self, operation_name, kwarg):
150165

151166
with patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call):
152167
invoke_handler("Update", {
153-
"SourceBucketName": "<source-bucket>",
154-
"SourceObjectKey": "<source-object-key>",
168+
"SourceBucketNames": ["<source-bucket>"],
169+
"SourceObjectKeys": ["<source-object-key>"],
155170
"DestinationBucketName": "<dest-bucket-name>",
156171
"DistributionId": "<cf-dist-id>"
157172
}, old_resource_props={
@@ -171,8 +186,8 @@ def mock_make_api_call(self, operation_name, kwarg):
171186

172187
with patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call):
173188
invoke_handler("Update", {
174-
"SourceBucketName": "<source-bucket>",
175-
"SourceObjectKey": "<source-object-key>",
189+
"SourceBucketNames": ["<source-bucket>"],
190+
"SourceObjectKeys": ["<source-object-key>"],
176191
"DestinationBucketName": "<dest-bucket-name>",
177192
"DestinationBucketKeyPrefix": "<dest-prefix>",
178193
"DistributionId": "<cf-dist-id>"
@@ -194,8 +209,8 @@ def mock_make_api_call(self, operation_name, kwarg):
194209

195210
with patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call):
196211
invoke_handler("Update", {
197-
"SourceBucketName": "<source-bucket>",
198-
"SourceObjectKey": "<source-object-key>",
212+
"SourceBucketNames": ["<source-bucket>"],
213+
"SourceObjectKeys": ["<source-object-key>"],
199214
"DestinationBucketName": "<dest-bucket-name>",
200215
"DistributionId": "<cf-dist-id>",
201216
"DistributionPaths": ["/path1/*", "/path2/*"]
@@ -205,8 +220,8 @@ def mock_make_api_call(self, operation_name, kwarg):
205220

206221
def test_update_new_dest_retain(self):
207222
invoke_handler("Update", {
208-
"SourceBucketName": "<source-bucket>",
209-
"SourceObjectKey": "<source-object-key>",
223+
"SourceBucketNames": ["<source-bucket>"],
224+
"SourceObjectKeys": ["<source-object-key>"],
210225
"DestinationBucketName": "<dest-bucket-name>",
211226
}, old_resource_props={
212227
"DestinationBucketName": "<dest-bucket-name>",
@@ -220,8 +235,8 @@ def test_update_new_dest_retain(self):
220235

221236
def test_update_new_dest_no_retain(self):
222237
invoke_handler("Update", {
223-
"SourceBucketName": "<source-bucket>",
224-
"SourceObjectKey": "<source-object-key>",
238+
"SourceBucketNames": ["<source-bucket>"],
239+
"SourceObjectKeys": ["<source-object-key>"],
225240
"DestinationBucketName": "<new-dest-bucket-name>",
226241
"RetainOnDelete": "false"
227242
}, old_resource_props={
@@ -238,8 +253,8 @@ def test_update_new_dest_no_retain(self):
238253

239254
def test_update_new_dest_retain_implicit(self):
240255
invoke_handler("Update", {
241-
"SourceBucketName": "<source-bucket>",
242-
"SourceObjectKey": "<source-object-key>",
256+
"SourceBucketNames": ["<source-bucket>"],
257+
"SourceObjectKeys": ["<source-object-key>"],
243258
"DestinationBucketName": "<new-dest-bucket-name>",
244259
}, old_resource_props={
245260
"DestinationBucketName": "<old-dest-bucket-name>",
@@ -253,8 +268,8 @@ def test_update_new_dest_retain_implicit(self):
253268

254269
def test_update_new_dest_prefix_no_retain(self):
255270
invoke_handler("Update", {
256-
"SourceBucketName": "<source-bucket>",
257-
"SourceObjectKey": "<source-object-key>",
271+
"SourceBucketNames": ["<source-bucket>"],
272+
"SourceObjectKeys": ["<source-object-key>"],
258273
"DestinationBucketName": "<dest-bucket-name>",
259274
"DestinationBucketKeyPrefix": "<new-dest-prefix>",
260275
"RetainOnDelete": "false"
@@ -271,8 +286,8 @@ def test_update_new_dest_prefix_no_retain(self):
271286

272287
def test_update_new_dest_prefix_retain_implicit(self):
273288
invoke_handler("Update", {
274-
"SourceBucketName": "<source-bucket>",
275-
"SourceObjectKey": "<source-object-key>",
289+
"SourceBucketNames": ["<source-bucket>"],
290+
"SourceObjectKeys": ["<source-object-key>"],
276291
"DestinationBucketName": "<dest-bucket-name>",
277292
"DestinationBucketKeyPrefix": "<new-dest-prefix>"
278293
}, old_resource_props={
@@ -290,8 +305,8 @@ def test_update_new_dest_prefix_retain_implicit(self):
290305

291306
def test_physical_id_allocated_on_create_and_reused_afterwards(self):
292307
create_resp = invoke_handler("Create", {
293-
"SourceBucketName": "<source-bucket>",
294-
"SourceObjectKey": "<source-object-key>",
308+
"SourceBucketNames": ["<source-bucket>"],
309+
"SourceObjectKeys": ["<source-object-key>"],
295310
"DestinationBucketName": "<dest-bucket-name>",
296311
})
297312

@@ -301,8 +316,8 @@ def test_physical_id_allocated_on_create_and_reused_afterwards(self):
301316
# now issue an update and pass in the physical id. expect the same
302317
# one to be returned back
303318
update_resp = invoke_handler("Update", {
304-
"SourceBucketName": "<source-bucket>",
305-
"SourceObjectKey": "<source-object-key>",
319+
"SourceBucketNames": ["<source-bucket>"],
320+
"SourceObjectKeys": ["<source-object-key>"],
306321
"DestinationBucketName": "<new-dest-bucket-name>",
307322
}, old_resource_props={
308323
"DestinationBucketName": "<dest-bucket-name>",
@@ -311,17 +326,17 @@ def test_physical_id_allocated_on_create_and_reused_afterwards(self):
311326

312327
# now issue a delete, and make sure this also applies
313328
delete_resp = invoke_handler("Delete", {
314-
"SourceBucketName": "<source-bucket>",
315-
"SourceObjectKey": "<source-object-key>",
329+
"SourceBucketNames": ["<source-bucket>"],
330+
"SourceObjectKeys": ["<source-object-key>"],
316331
"DestinationBucketName": "<dest-bucket-name>",
317332
"RetainOnDelete": "false"
318333
}, physical_id=phid)
319334
self.assertEqual(delete_resp['PhysicalResourceId'], phid)
320335

321336
def test_fails_when_physical_id_not_present_in_update(self):
322337
update_resp = invoke_handler("Update", {
323-
"SourceBucketName": "<source-bucket>",
324-
"SourceObjectKey": "<source-object-key>",
338+
"SourceBucketNames": ["<source-bucket>"],
339+
"SourceObjectKeys": ["<source-object-key>"],
325340
"DestinationBucketName": "<new-dest-bucket-name>",
326341
}, old_resource_props={
327342
"DestinationBucketName": "<dest-bucket-name>",
@@ -331,8 +346,8 @@ def test_fails_when_physical_id_not_present_in_update(self):
331346

332347
def test_fails_when_physical_id_not_present_in_delete(self):
333348
update_resp = invoke_handler("Delete", {
334-
"SourceBucketName": "<source-bucket>",
335-
"SourceObjectKey": "<source-object-key>",
349+
"SourceBucketNames": ["<source-bucket>"],
350+
"SourceObjectKeys": ["<source-object-key>"],
336351
"DestinationBucketName": "<new-dest-bucket-name>",
337352
}, old_resource_props={
338353
"DestinationBucketName": "<dest-bucket-name>",

0 commit comments

Comments
 (0)